From d8ca80955ab89f5f4ec4b49c356fff3f83f8ce90 Mon Sep 17 00:00:00 2001 From: Donovan Jimenez Date: Tue, 10 Apr 2012 18:30:42 -0400 Subject: [PATCH] go1 updates: standard errors now in syscall package, no more makefile support --- Makefile | 23 ----------------------- README.md | 26 ++++++-------------------- converter.go | 37 +++++++++++++++++++------------------ iconv.go | 26 ++++---------------------- iconv_test.go | 43 ++++++++++++++++++++++--------------------- reader.go | 22 +++++++++++----------- writer.go | 33 +++++++++++++++------------------ 7 files changed, 77 insertions(+), 133 deletions(-) delete mode 100644 Makefile diff --git a/Makefile b/Makefile deleted file mode 100644 index 992841b..0000000 --- a/Makefile +++ /dev/null @@ -1,23 +0,0 @@ -# standard GO make file preamble -include $(GOROOT)/src/Make.inc - -# target package name -TARG=iconv - -# regular go files -GOFILES=\ - reader.go\ - writer.go\ - -# files that must be processed by cgo -CGOFILES=\ - converter.go\ - iconv.go\ - -# on non glibc systems, we usually need to load the library -ifneq ($(GOOS),linux) -CGO_LDFLAGS=-liconv -endif - -# standard GO make file include for packages -include $(GOROOT)/src/Make.pkg \ No newline at end of file diff --git a/README.md b/README.md index 04cf03d..d9be2c2 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,14 @@ # Install -The main method of installation is through gomake (provided in $GOROOT/bin) +The main method of installation is through "go install" (provided in $GOROOT/bin) - git clone git://github.com/djimenez/iconv.go.git iconv - cd iconv - gomake install - -Alternatively, you can try using goinstall (also provided in $GOROOT/bin). -However, because iconv.go uses cgo to wrap iconv functions, the build may not -succeed on all systems. At time of writing goinstall was still experimental and -has known issues with cgo based packages because of how it produces its own -make file. - - goinstall github.com/djimenez/iconv.go + go install github.com/djimenez/iconv.go # Usage To use the package, you'll need the appropriate import statement: import ( - // if you used gomake install directly, you'll want this import - iconv - - // if you used goinstall, you'll want this import iconv "github.com/djimenez/iconv.go" ) @@ -96,18 +82,18 @@ automatically do this since it can be used to process a full stream in chunks. So you'll need to remember to pass a nil input buffer at the end yourself, just like you would with direct iconv usage. -## Converting an *io.Reader +## Converting an \*io.Reader -The iconv.Reader allows any other *io.Reader to be wrapped and have its bytes +The iconv.Reader allows any other \*io.Reader to be wrapped and have its bytes transcoded as they are read. // We're wrapping stdin for simplicity, but a File or network reader could // be wrapped as well reader,_ := iconv.NewReader(os.Stdin, "utf-8", "windows-1252") -## Converting an *io.Writer +## Converting an \*io.Writer -The iconv.Writer allows any other *io.Writer to be wrapped and have its bytes +The iconv.Writer allows any other \*io.Writer to be wrapped and have its bytes transcoded as they are written. // We're wrapping stdout for simplicity, but a File or network reader could diff --git a/converter.go b/converter.go index 5567161..540b90c 100644 --- a/converter.go +++ b/converter.go @@ -1,22 +1,23 @@ package iconv /* +#cgo darwin LDFLAGS: -liconv #include #include */ import "C" -import "os" +import "syscall" import "unsafe" type Converter struct { context C.iconv_t - open bool + open bool } // Initialize a new Converter. If fromEncoding or toEncoding are not supported by // iconv then an EINVAL error will be returned. An ENOMEM error maybe returned if // there is not enough memory to initialize an iconv descriptor -func NewConverter(fromEncoding string, toEncoding string) (converter *Converter, err Error) { +func NewConverter(fromEncoding string, toEncoding string) (converter *Converter, err error) { converter = new(Converter) // convert to C strings @@ -45,7 +46,7 @@ func (this *Converter) destroy() { } // Close a Converter's iconv description explicitly -func (this *Converter) Close() (err os.Error) { +func (this *Converter) Close() (err error) { if this.open { _, err = C.iconv_close(this.context) } @@ -64,7 +65,7 @@ func (this *Converter) Close() (err os.Error) { // For shift based output encodings, any end shift byte sequences can be generated by // passing a 0 length byte slice as input. Also passing a 0 length byte slice for output // will simply reset the iconv descriptor shift state without writing any bytes. -func (this *Converter) Convert(input []byte, output []byte) (bytesRead int, bytesWritten int, err Error) { +func (this *Converter) Convert(input []byte, output []byte) (bytesRead int, bytesWritten int, err error) { // make sure we are still open if this.open { inputLeft := C.size_t(len(input)) @@ -77,7 +78,7 @@ func (this *Converter) Convert(input []byte, output []byte) (bytesRead int, byte inputPointer := (*C.char)(unsafe.Pointer(&input[0])) outputPointer := (*C.char)(unsafe.Pointer(&output[0])) - _,err = C.iconv(this.context, &inputPointer, &inputLeft, &outputPointer, &outputLeft) + _, err = C.iconv(this.context, &inputPointer, &inputLeft, &outputPointer, &outputLeft) // update byte counters bytesRead = len(input) - int(inputLeft) @@ -86,16 +87,16 @@ func (this *Converter) Convert(input []byte, output []byte) (bytesRead int, byte // inputPointer will be nil, outputPointer is generated as above outputPointer := (*C.char)(unsafe.Pointer(&output[0])) - _,err = C.iconv(this.context, nil, &inputLeft, &outputPointer, &outputLeft) + _, err = C.iconv(this.context, nil, &inputLeft, &outputPointer, &outputLeft) // update write byte counter bytesWritten = len(output) - int(outputLeft) } else { // both input and output are zero length, do a shift state reset - _,err = C.iconv(this.context, nil, &inputLeft, nil, &outputLeft) + _, err = C.iconv(this.context, nil, &inputLeft, nil, &outputLeft) } } else { - err = EBADF + err = syscall.EBADF } return bytesRead, bytesWritten, err @@ -105,12 +106,12 @@ func (this *Converter) Convert(input []byte, output []byte) (bytesRead int, byte // // EILSEQ error may be returned if input contains invalid bytes for the // Converter's fromEncoding. -func (this *Converter) ConvertString(input string) (output string, err Error) { +func (this *Converter) ConvertString(input string) (output string, err error) { // make sure we are still open if this.open { // construct the buffers inputBuffer := []byte(input) - outputBuffer := make([]byte, len(inputBuffer) * 2) // we use a larger buffer to help avoid resizing later + outputBuffer := make([]byte, len(inputBuffer)*2) // we use a larger buffer to help avoid resizing later // call Convert until all input bytes are read or an error occurs var bytesRead, totalBytesRead, bytesWritten, totalBytesWritten int @@ -124,11 +125,11 @@ func (this *Converter) ConvertString(input string) (output string, err Error) { // check for the E2BIG error specifically, we can add to the output // buffer to correct for it and then continue - if err == E2BIG { + if err == syscall.E2BIG { // increase the size of the output buffer by another input length // first, create a new buffer - tempBuffer := make([]byte, len(outputBuffer) + len(inputBuffer)) - + tempBuffer := make([]byte, len(outputBuffer)+len(inputBuffer)) + // copy the existing data copy(tempBuffer, outputBuffer) @@ -139,19 +140,19 @@ func (this *Converter) ConvertString(input string) (output string, err Error) { err = nil } } - + if err == nil { // perform a final shift state reset _, bytesWritten, err = this.Convert([]byte{}, outputBuffer[totalBytesWritten:]) - + // update total count totalBytesWritten += bytesWritten } - + // construct the final output string output = string(outputBuffer[:totalBytesWritten]) } else { - err = EBADF + err = syscall.EBADF } return output, err diff --git a/iconv.go b/iconv.go index 4664109..3e0036a 100644 --- a/iconv.go +++ b/iconv.go @@ -5,26 +5,8 @@ some convenient interface implementations like a Reader and Writer. */ package iconv -/* -#include -*/ -import "C" -import "os" - -// Alias os.Error for convenience -type Error os.Error - -// Error codes returned from iconv functions -var ( - E2BIG Error = os.Errno(int(C.E2BIG)) - EBADF Error = os.Errno(int(C.EBADF)) - EINVAL Error = os.Errno(int(C.EINVAL)) - EILSEQ Error = os.Errno(int(C.EILSEQ)) - ENOMEM Error = os.Errno(int(C.ENOMEM)) -) - // All in one Convert method, rather than requiring the construction of an iconv.Converter -func Convert(input []byte, output []byte, fromEncoding string, toEncoding string) (bytesRead int, bytesWritten int, err Error) { +func Convert(input []byte, output []byte, fromEncoding string, toEncoding string) (bytesRead int, bytesWritten int, err error) { // create a temporary converter converter, err := NewConverter(fromEncoding, toEncoding) @@ -34,10 +16,10 @@ func Convert(input []byte, output []byte, fromEncoding string, toEncoding string if err == nil { var shiftBytesWritten int - + // call Convert with a nil input to generate any end shift sequences _, shiftBytesWritten, err = converter.Convert(nil, output[bytesWritten:]) - + // add shift bytes to total bytes bytesWritten += shiftBytesWritten } @@ -50,7 +32,7 @@ func Convert(input []byte, output []byte, fromEncoding string, toEncoding string } // All in one ConvertString method, rather than requiring the construction of an iconv.Converter -func ConvertString(input string, fromEncoding string, toEncoding string) (output string, err Error) { +func ConvertString(input string, fromEncoding string, toEncoding string) (output string, err error) { // create a temporary converter converter, err := NewConverter(fromEncoding, toEncoding) diff --git a/iconv_test.go b/iconv_test.go index f976ded..fed0869 100644 --- a/iconv_test.go +++ b/iconv_test.go @@ -1,21 +1,22 @@ package iconv import ( + "syscall" "testing" ) type iconvTest struct { - description string - input string - inputEncoding string - output string + description string + input string + inputEncoding string + output string outputEncoding string - bytesRead int - bytesWritten int - err Error + bytesRead int + bytesWritten int + err error } -var iconvTests = []iconvTest { +var iconvTests = []iconvTest{ iconvTest{ "simple utf-8 to latin1 conversion success", "Hello World!", "utf-8", @@ -26,25 +27,25 @@ var iconvTests = []iconvTest { "invalid source encoding causes EINVAL", "", "doesnotexist", "", "utf-8", - 0, 0, EINVAL, + 0, 0, syscall.EINVAL, }, iconvTest{ "invalid destination encoding causes EINVAL", "", "utf-8", "", "doesnotexist", - 0, 0, EINVAL, + 0, 0, syscall.EINVAL, }, iconvTest{ "invalid input sequence causes EILSEQ", "\xFF", "utf-8", "", "latin1", - 0, 0, EILSEQ, + 0, 0, syscall.EILSEQ, }, iconvTest{ "invalid input causes partial output and EILSEQ", "Hello\xFF", "utf-8", "Hello", "latin1", - 5, 5, EILSEQ, + 5, 5, syscall.EILSEQ, }, } @@ -52,12 +53,12 @@ func TestConvertString(t *testing.T) { for _, test := range iconvTests { // perform the conversion output, err := ConvertString(test.input, test.inputEncoding, test.outputEncoding) - + // check that output and err match if output != test.output { t.Errorf("test \"%s\" failed, output did not match expected", test.description) } - + // check that err is same as expected if err != test.err { if test.err != nil { @@ -77,29 +78,29 @@ func TestConvert(t *testing.T) { for _, test := range iconvTests { // setup input buffer input := []byte(test.input) - + // setup a buffer as large as the expected bytesWritten output := make([]byte, 50) - + // peform the conversion bytesRead, bytesWritten, err := Convert(input, output, test.inputEncoding, test.outputEncoding) - + // check that bytesRead is same as expected if bytesRead != test.bytesRead { t.Errorf("test \"%s\" failed, bytesRead did not match expected", test.description) } - + // check that bytesWritten is same as expected if bytesWritten != test.bytesWritten { t.Errorf("test \"%s\" failed, bytesWritten did not match expected", test.description) } - + // check output bytes against expected - simplest to convert output to // string and then do an equality check which is actually a byte wise operation if string(output[:bytesWritten]) != test.output { t.Errorf("test \"%s\" failed, output did not match expected", test.description) } - + // check that err is same as expected if err != test.err { if test.err != nil { @@ -113,4 +114,4 @@ func TestConvert(t *testing.T) { } } } -} \ No newline at end of file +} diff --git a/reader.go b/reader.go index 2d9bca1..2835ce6 100644 --- a/reader.go +++ b/reader.go @@ -1,19 +1,19 @@ package iconv -import ( +import ( "io" - "os" + "syscall" ) type Reader struct { - source io.Reader - converter *Converter - buffer []byte + source io.Reader + converter *Converter + buffer []byte readPos, writePos int - err os.Error + err error } -func NewReader(source io.Reader, fromEncoding string, toEncoding string) (*Reader, os.Error) { +func NewReader(source io.Reader, fromEncoding string, toEncoding string) (*Reader, error) { // create a converter converter, err := NewConverter(fromEncoding, toEncoding) @@ -33,7 +33,7 @@ func NewReaderFromConverter(source io.Reader, converter *Converter) (reader *Rea reader.converter = converter // create 8K buffers - reader.buffer = make([]byte, 8 * 1024) + reader.buffer = make([]byte, 8*1024) return reader } @@ -62,7 +62,7 @@ func (this *Reader) fillBuffer() { } // implement the io.Reader interface -func (this *Reader) Read(p []byte) (n int, err os.Error) { +func (this *Reader) Read(p []byte) (n int, err error) { // checks for when we have no data for this.writePos == 0 || this.readPos == this.writePos { // if we have an error / EOF, just return it @@ -72,7 +72,7 @@ func (this *Reader) Read(p []byte) (n int, err os.Error) { // else, fill our buffer this.fillBuffer() - } + } // TODO: checks for when we have less data than len(p) @@ -89,7 +89,7 @@ func (this *Reader) Read(p []byte) (n int, err os.Error) { // as at least 1 byte was written. If we experienced an E2BIG // and no bytes were written then the buffer is too small for // even the next character - if err != E2BIG || bytesWritten == 0 { + if err != syscall.E2BIG || bytesWritten == 0 { // track anything else this.err = err } diff --git a/writer.go b/writer.go index a8f8370..df7c062 100644 --- a/writer.go +++ b/writer.go @@ -1,19 +1,16 @@ package iconv -import ( - "io" - "os" -) +import "io" type Writer struct { - destination io.Writer - converter *Converter - buffer []byte + destination io.Writer + converter *Converter + buffer []byte readPos, writePos int - err os.Error + err error } -func NewWriter(destination io.Writer, fromEncoding string, toEncoding string) (*Writer, os.Error) { +func NewWriter(destination io.Writer, fromEncoding string, toEncoding string) (*Writer, error) { // create a converter converter, err := NewConverter(fromEncoding, toEncoding) @@ -33,7 +30,7 @@ func NewWriterFromConverter(destination io.Writer, converter *Converter) (writer writer.converter = converter // create 8K buffers - writer.buffer = make([]byte, 8 * 1024) + writer.buffer = make([]byte, 8*1024) return writer } @@ -41,10 +38,10 @@ func NewWriterFromConverter(destination io.Writer, converter *Converter) (writer func (this *Writer) emptyBuffer() { // write new data out of buffer bytesWritten, err := this.destination.Write(this.buffer[this.readPos:this.writePos]) - + // update read position this.readPos += bytesWritten - + // slide existing data to beginning if this.readPos > 0 { // copy current bytes - is this guaranteed safe? @@ -62,24 +59,24 @@ func (this *Writer) emptyBuffer() { } // implement the io.Writer interface -func (this *Writer) Write(p []byte) (n int, err os.Error) { +func (this *Writer) Write(p []byte) (n int, err error) { // write data into our internal buffer bytesRead, bytesWritten, err := this.converter.Convert(p, this.buffer[this.writePos:]) - + // update bytes written for return n += bytesRead this.writePos += bytesWritten - + // checks for when we have a full buffer for this.writePos > 0 { // if we have an error, just return it if this.err != nil { - return + return } - + // else empty the buffer this.emptyBuffer() } - + return n, err }