@@ -1,23 +0,0 @@ | |||
# standard GO make file preamble | |||
include $(GOROOT)/src/Make.inc | |||
# target package name | |||
TARG=iconv | |||
# regular go files | |||
GOFILES=\ | |||
reader.go\ | |||
writer.go\ | |||
# files that must be processed by cgo | |||
CGOFILES=\ | |||
converter.go\ | |||
iconv.go\ | |||
# on non glibc systems, we usually need to load the library | |||
ifneq ($(GOOS),linux) | |||
CGO_LDFLAGS=-liconv | |||
endif | |||
# standard GO make file include for packages | |||
include $(GOROOT)/src/Make.pkg |
@@ -1,28 +1,14 @@ | |||
# Install | |||
The main method of installation is through gomake (provided in $GOROOT/bin) | |||
The main method of installation is through "go install" (provided in $GOROOT/bin) | |||
git clone git://github.com/djimenez/iconv.go.git iconv | |||
cd iconv | |||
gomake install | |||
Alternatively, you can try using goinstall (also provided in $GOROOT/bin). | |||
However, because iconv.go uses cgo to wrap iconv functions, the build may not | |||
succeed on all systems. At time of writing goinstall was still experimental and | |||
has known issues with cgo based packages because of how it produces its own | |||
make file. | |||
goinstall github.com/djimenez/iconv.go | |||
go install github.com/djimenez/iconv.go | |||
# Usage | |||
To use the package, you'll need the appropriate import statement: | |||
import ( | |||
// if you used gomake install directly, you'll want this import | |||
iconv | |||
// if you used goinstall, you'll want this import | |||
iconv "github.com/djimenez/iconv.go" | |||
) | |||
@@ -96,18 +82,18 @@ automatically do this since it can be used to process a full stream in chunks. | |||
So you'll need to remember to pass a nil input buffer at the end yourself, just | |||
like you would with direct iconv usage. | |||
## Converting an *io.Reader | |||
## Converting an \*io.Reader | |||
The iconv.Reader allows any other *io.Reader to be wrapped and have its bytes | |||
The iconv.Reader allows any other \*io.Reader to be wrapped and have its bytes | |||
transcoded as they are read. | |||
// We're wrapping stdin for simplicity, but a File or network reader could | |||
// be wrapped as well | |||
reader,_ := iconv.NewReader(os.Stdin, "utf-8", "windows-1252") | |||
## Converting an *io.Writer | |||
## Converting an \*io.Writer | |||
The iconv.Writer allows any other *io.Writer to be wrapped and have its bytes | |||
The iconv.Writer allows any other \*io.Writer to be wrapped and have its bytes | |||
transcoded as they are written. | |||
// We're wrapping stdout for simplicity, but a File or network reader could | |||
@@ -1,22 +1,23 @@ | |||
package iconv | |||
/* | |||
#cgo darwin LDFLAGS: -liconv | |||
#include <stdlib.h> | |||
#include <iconv.h> | |||
*/ | |||
import "C" | |||
import "os" | |||
import "syscall" | |||
import "unsafe" | |||
type Converter struct { | |||
context C.iconv_t | |||
open bool | |||
open bool | |||
} | |||
// Initialize a new Converter. If fromEncoding or toEncoding are not supported by | |||
// iconv then an EINVAL error will be returned. An ENOMEM error maybe returned if | |||
// there is not enough memory to initialize an iconv descriptor | |||
func NewConverter(fromEncoding string, toEncoding string) (converter *Converter, err Error) { | |||
func NewConverter(fromEncoding string, toEncoding string) (converter *Converter, err error) { | |||
converter = new(Converter) | |||
// convert to C strings | |||
@@ -45,7 +46,7 @@ func (this *Converter) destroy() { | |||
} | |||
// Close a Converter's iconv description explicitly | |||
func (this *Converter) Close() (err os.Error) { | |||
func (this *Converter) Close() (err error) { | |||
if this.open { | |||
_, err = C.iconv_close(this.context) | |||
} | |||
@@ -64,7 +65,7 @@ func (this *Converter) Close() (err os.Error) { | |||
// For shift based output encodings, any end shift byte sequences can be generated by | |||
// passing a 0 length byte slice as input. Also passing a 0 length byte slice for output | |||
// will simply reset the iconv descriptor shift state without writing any bytes. | |||
func (this *Converter) Convert(input []byte, output []byte) (bytesRead int, bytesWritten int, err Error) { | |||
func (this *Converter) Convert(input []byte, output []byte) (bytesRead int, bytesWritten int, err error) { | |||
// make sure we are still open | |||
if this.open { | |||
inputLeft := C.size_t(len(input)) | |||
@@ -77,7 +78,7 @@ func (this *Converter) Convert(input []byte, output []byte) (bytesRead int, byte | |||
inputPointer := (*C.char)(unsafe.Pointer(&input[0])) | |||
outputPointer := (*C.char)(unsafe.Pointer(&output[0])) | |||
_,err = C.iconv(this.context, &inputPointer, &inputLeft, &outputPointer, &outputLeft) | |||
_, err = C.iconv(this.context, &inputPointer, &inputLeft, &outputPointer, &outputLeft) | |||
// update byte counters | |||
bytesRead = len(input) - int(inputLeft) | |||
@@ -86,16 +87,16 @@ func (this *Converter) Convert(input []byte, output []byte) (bytesRead int, byte | |||
// inputPointer will be nil, outputPointer is generated as above | |||
outputPointer := (*C.char)(unsafe.Pointer(&output[0])) | |||
_,err = C.iconv(this.context, nil, &inputLeft, &outputPointer, &outputLeft) | |||
_, err = C.iconv(this.context, nil, &inputLeft, &outputPointer, &outputLeft) | |||
// update write byte counter | |||
bytesWritten = len(output) - int(outputLeft) | |||
} else { | |||
// both input and output are zero length, do a shift state reset | |||
_,err = C.iconv(this.context, nil, &inputLeft, nil, &outputLeft) | |||
_, err = C.iconv(this.context, nil, &inputLeft, nil, &outputLeft) | |||
} | |||
} else { | |||
err = EBADF | |||
err = syscall.EBADF | |||
} | |||
return bytesRead, bytesWritten, err | |||
@@ -105,12 +106,12 @@ func (this *Converter) Convert(input []byte, output []byte) (bytesRead int, byte | |||
// | |||
// EILSEQ error may be returned if input contains invalid bytes for the | |||
// Converter's fromEncoding. | |||
func (this *Converter) ConvertString(input string) (output string, err Error) { | |||
func (this *Converter) ConvertString(input string) (output string, err error) { | |||
// make sure we are still open | |||
if this.open { | |||
// construct the buffers | |||
inputBuffer := []byte(input) | |||
outputBuffer := make([]byte, len(inputBuffer) * 2) // we use a larger buffer to help avoid resizing later | |||
outputBuffer := make([]byte, len(inputBuffer)*2) // we use a larger buffer to help avoid resizing later | |||
// call Convert until all input bytes are read or an error occurs | |||
var bytesRead, totalBytesRead, bytesWritten, totalBytesWritten int | |||
@@ -124,11 +125,11 @@ func (this *Converter) ConvertString(input string) (output string, err Error) { | |||
// check for the E2BIG error specifically, we can add to the output | |||
// buffer to correct for it and then continue | |||
if err == E2BIG { | |||
if err == syscall.E2BIG { | |||
// increase the size of the output buffer by another input length | |||
// first, create a new buffer | |||
tempBuffer := make([]byte, len(outputBuffer) + len(inputBuffer)) | |||
tempBuffer := make([]byte, len(outputBuffer)+len(inputBuffer)) | |||
// copy the existing data | |||
copy(tempBuffer, outputBuffer) | |||
@@ -139,19 +140,19 @@ func (this *Converter) ConvertString(input string) (output string, err Error) { | |||
err = nil | |||
} | |||
} | |||
if err == nil { | |||
// perform a final shift state reset | |||
_, bytesWritten, err = this.Convert([]byte{}, outputBuffer[totalBytesWritten:]) | |||
// update total count | |||
totalBytesWritten += bytesWritten | |||
} | |||
// construct the final output string | |||
output = string(outputBuffer[:totalBytesWritten]) | |||
} else { | |||
err = EBADF | |||
err = syscall.EBADF | |||
} | |||
return output, err | |||
@@ -5,26 +5,8 @@ some convenient interface implementations like a Reader and Writer. | |||
*/ | |||
package iconv | |||
/* | |||
#include <errno.h> | |||
*/ | |||
import "C" | |||
import "os" | |||
// Alias os.Error for convenience | |||
type Error os.Error | |||
// Error codes returned from iconv functions | |||
var ( | |||
E2BIG Error = os.Errno(int(C.E2BIG)) | |||
EBADF Error = os.Errno(int(C.EBADF)) | |||
EINVAL Error = os.Errno(int(C.EINVAL)) | |||
EILSEQ Error = os.Errno(int(C.EILSEQ)) | |||
ENOMEM Error = os.Errno(int(C.ENOMEM)) | |||
) | |||
// All in one Convert method, rather than requiring the construction of an iconv.Converter | |||
func Convert(input []byte, output []byte, fromEncoding string, toEncoding string) (bytesRead int, bytesWritten int, err Error) { | |||
func Convert(input []byte, output []byte, fromEncoding string, toEncoding string) (bytesRead int, bytesWritten int, err error) { | |||
// create a temporary converter | |||
converter, err := NewConverter(fromEncoding, toEncoding) | |||
@@ -34,10 +16,10 @@ func Convert(input []byte, output []byte, fromEncoding string, toEncoding string | |||
if err == nil { | |||
var shiftBytesWritten int | |||
// call Convert with a nil input to generate any end shift sequences | |||
_, shiftBytesWritten, err = converter.Convert(nil, output[bytesWritten:]) | |||
// add shift bytes to total bytes | |||
bytesWritten += shiftBytesWritten | |||
} | |||
@@ -50,7 +32,7 @@ func Convert(input []byte, output []byte, fromEncoding string, toEncoding string | |||
} | |||
// All in one ConvertString method, rather than requiring the construction of an iconv.Converter | |||
func ConvertString(input string, fromEncoding string, toEncoding string) (output string, err Error) { | |||
func ConvertString(input string, fromEncoding string, toEncoding string) (output string, err error) { | |||
// create a temporary converter | |||
converter, err := NewConverter(fromEncoding, toEncoding) | |||
@@ -1,21 +1,22 @@ | |||
package iconv | |||
import ( | |||
"syscall" | |||
"testing" | |||
) | |||
type iconvTest struct { | |||
description string | |||
input string | |||
inputEncoding string | |||
output string | |||
description string | |||
input string | |||
inputEncoding string | |||
output string | |||
outputEncoding string | |||
bytesRead int | |||
bytesWritten int | |||
err Error | |||
bytesRead int | |||
bytesWritten int | |||
err error | |||
} | |||
var iconvTests = []iconvTest { | |||
var iconvTests = []iconvTest{ | |||
iconvTest{ | |||
"simple utf-8 to latin1 conversion success", | |||
"Hello World!", "utf-8", | |||
@@ -26,25 +27,25 @@ var iconvTests = []iconvTest { | |||
"invalid source encoding causes EINVAL", | |||
"", "doesnotexist", | |||
"", "utf-8", | |||
0, 0, EINVAL, | |||
0, 0, syscall.EINVAL, | |||
}, | |||
iconvTest{ | |||
"invalid destination encoding causes EINVAL", | |||
"", "utf-8", | |||
"", "doesnotexist", | |||
0, 0, EINVAL, | |||
0, 0, syscall.EINVAL, | |||
}, | |||
iconvTest{ | |||
"invalid input sequence causes EILSEQ", | |||
"\xFF", "utf-8", | |||
"", "latin1", | |||
0, 0, EILSEQ, | |||
0, 0, syscall.EILSEQ, | |||
}, | |||
iconvTest{ | |||
"invalid input causes partial output and EILSEQ", | |||
"Hello\xFF", "utf-8", | |||
"Hello", "latin1", | |||
5, 5, EILSEQ, | |||
5, 5, syscall.EILSEQ, | |||
}, | |||
} | |||
@@ -52,12 +53,12 @@ func TestConvertString(t *testing.T) { | |||
for _, test := range iconvTests { | |||
// perform the conversion | |||
output, err := ConvertString(test.input, test.inputEncoding, test.outputEncoding) | |||
// check that output and err match | |||
if output != test.output { | |||
t.Errorf("test \"%s\" failed, output did not match expected", test.description) | |||
} | |||
// check that err is same as expected | |||
if err != test.err { | |||
if test.err != nil { | |||
@@ -77,29 +78,29 @@ func TestConvert(t *testing.T) { | |||
for _, test := range iconvTests { | |||
// setup input buffer | |||
input := []byte(test.input) | |||
// setup a buffer as large as the expected bytesWritten | |||
output := make([]byte, 50) | |||
// peform the conversion | |||
bytesRead, bytesWritten, err := Convert(input, output, test.inputEncoding, test.outputEncoding) | |||
// check that bytesRead is same as expected | |||
if bytesRead != test.bytesRead { | |||
t.Errorf("test \"%s\" failed, bytesRead did not match expected", test.description) | |||
} | |||
// check that bytesWritten is same as expected | |||
if bytesWritten != test.bytesWritten { | |||
t.Errorf("test \"%s\" failed, bytesWritten did not match expected", test.description) | |||
} | |||
// check output bytes against expected - simplest to convert output to | |||
// string and then do an equality check which is actually a byte wise operation | |||
if string(output[:bytesWritten]) != test.output { | |||
t.Errorf("test \"%s\" failed, output did not match expected", test.description) | |||
} | |||
// check that err is same as expected | |||
if err != test.err { | |||
if test.err != nil { | |||
@@ -113,4 +114,4 @@ func TestConvert(t *testing.T) { | |||
} | |||
} | |||
} | |||
} | |||
} |
@@ -1,19 +1,19 @@ | |||
package iconv | |||
import ( | |||
import ( | |||
"io" | |||
"os" | |||
"syscall" | |||
) | |||
type Reader struct { | |||
source io.Reader | |||
converter *Converter | |||
buffer []byte | |||
source io.Reader | |||
converter *Converter | |||
buffer []byte | |||
readPos, writePos int | |||
err os.Error | |||
err error | |||
} | |||
func NewReader(source io.Reader, fromEncoding string, toEncoding string) (*Reader, os.Error) { | |||
func NewReader(source io.Reader, fromEncoding string, toEncoding string) (*Reader, error) { | |||
// create a converter | |||
converter, err := NewConverter(fromEncoding, toEncoding) | |||
@@ -33,7 +33,7 @@ func NewReaderFromConverter(source io.Reader, converter *Converter) (reader *Rea | |||
reader.converter = converter | |||
// create 8K buffers | |||
reader.buffer = make([]byte, 8 * 1024) | |||
reader.buffer = make([]byte, 8*1024) | |||
return reader | |||
} | |||
@@ -62,7 +62,7 @@ func (this *Reader) fillBuffer() { | |||
} | |||
// implement the io.Reader interface | |||
func (this *Reader) Read(p []byte) (n int, err os.Error) { | |||
func (this *Reader) Read(p []byte) (n int, err error) { | |||
// checks for when we have no data | |||
for this.writePos == 0 || this.readPos == this.writePos { | |||
// if we have an error / EOF, just return it | |||
@@ -72,7 +72,7 @@ func (this *Reader) Read(p []byte) (n int, err os.Error) { | |||
// else, fill our buffer | |||
this.fillBuffer() | |||
} | |||
} | |||
// TODO: checks for when we have less data than len(p) | |||
@@ -89,7 +89,7 @@ func (this *Reader) Read(p []byte) (n int, err os.Error) { | |||
// as at least 1 byte was written. If we experienced an E2BIG | |||
// and no bytes were written then the buffer is too small for | |||
// even the next character | |||
if err != E2BIG || bytesWritten == 0 { | |||
if err != syscall.E2BIG || bytesWritten == 0 { | |||
// track anything else | |||
this.err = err | |||
} | |||
@@ -1,19 +1,16 @@ | |||
package iconv | |||
import ( | |||
"io" | |||
"os" | |||
) | |||
import "io" | |||
type Writer struct { | |||
destination io.Writer | |||
converter *Converter | |||
buffer []byte | |||
destination io.Writer | |||
converter *Converter | |||
buffer []byte | |||
readPos, writePos int | |||
err os.Error | |||
err error | |||
} | |||
func NewWriter(destination io.Writer, fromEncoding string, toEncoding string) (*Writer, os.Error) { | |||
func NewWriter(destination io.Writer, fromEncoding string, toEncoding string) (*Writer, error) { | |||
// create a converter | |||
converter, err := NewConverter(fromEncoding, toEncoding) | |||
@@ -33,7 +30,7 @@ func NewWriterFromConverter(destination io.Writer, converter *Converter) (writer | |||
writer.converter = converter | |||
// create 8K buffers | |||
writer.buffer = make([]byte, 8 * 1024) | |||
writer.buffer = make([]byte, 8*1024) | |||
return writer | |||
} | |||
@@ -41,10 +38,10 @@ func NewWriterFromConverter(destination io.Writer, converter *Converter) (writer | |||
func (this *Writer) emptyBuffer() { | |||
// write new data out of buffer | |||
bytesWritten, err := this.destination.Write(this.buffer[this.readPos:this.writePos]) | |||
// update read position | |||
this.readPos += bytesWritten | |||
// slide existing data to beginning | |||
if this.readPos > 0 { | |||
// copy current bytes - is this guaranteed safe? | |||
@@ -62,24 +59,24 @@ func (this *Writer) emptyBuffer() { | |||
} | |||
// implement the io.Writer interface | |||
func (this *Writer) Write(p []byte) (n int, err os.Error) { | |||
func (this *Writer) Write(p []byte) (n int, err error) { | |||
// write data into our internal buffer | |||
bytesRead, bytesWritten, err := this.converter.Convert(p, this.buffer[this.writePos:]) | |||
// update bytes written for return | |||
n += bytesRead | |||
this.writePos += bytesWritten | |||
// checks for when we have a full buffer | |||
for this.writePos > 0 { | |||
// if we have an error, just return it | |||
if this.err != nil { | |||
return | |||
return | |||
} | |||
// else empty the buffer | |||
this.emptyBuffer() | |||
} | |||
return n, err | |||
} |