GaoreGo/iconv-go
7
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

go1 updates: standard errors now in syscall package, no more makefile support

Browse Source
This commit is contained in:
Donovan Jimenez 2012-04-10 18:30:42 -04:00
parent 8c9fe240c5
commit d8ca80955a
7 changed files with 77 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
Makefile
View File

@ -1,23 +0,0 @@
# standard GO make file preamble
include $(GOROOT)/src/Make.inc
# target package name
TARG=iconv
# regular go files
GOFILES=\
reader.go\
writer.go\
# files that must be processed by cgo
CGOFILES=\
converter.go\
iconv.go\
# on non glibc systems, we usually need to load the library
ifneq ($(GOOS),linux)
CGO_LDFLAGS=-liconv
endif
# standard GO make file include for packages
include $(GOROOT)/src/Make.pkg

26
README.md
View File

@ -1,28 +1,14 @@
# Install
The main method of installation is through gomake (provided in $GOROOT/bin)
The main method of installation is through "go install" (provided in $GOROOT/bin)
git clone git://github.com/djimenez/iconv.go.git iconv
cd iconv
gomake install
Alternatively, you can try using goinstall (also provided in $GOROOT/bin).
However, because iconv.go uses cgo to wrap iconv functions, the build may not
succeed on all systems. At time of writing goinstall was still experimental and
has known issues with cgo based packages because of how it produces its own
make file.
goinstall github.com/djimenez/iconv.go
go install github.com/djimenez/iconv.go
# Usage
To use the package, you'll need the appropriate import statement:
import (
// if you used gomake install directly, you'll want this import
iconv
// if you used goinstall, you'll want this import
iconv "github.com/djimenez/iconv.go"
)
@ -96,18 +82,18 @@ automatically do this since it can be used to process a full stream in chunks.
So you'll need to remember to pass a nil input buffer at the end yourself, just
like you would with direct iconv usage.
## Converting an *io.Reader
## Converting an \*io.Reader
The iconv.Reader allows any other *io.Reader to be wrapped and have its bytes
The iconv.Reader allows any other \*io.Reader to be wrapped and have its bytes
transcoded as they are read.
// We're wrapping stdin for simplicity, but a File or network reader could
// be wrapped as well
reader,_ := iconv.NewReader(os.Stdin, "utf-8", "windows-1252")
## Converting an *io.Writer
## Converting an \*io.Writer
The iconv.Writer allows any other *io.Writer to be wrapped and have its bytes
The iconv.Writer allows any other \*io.Writer to be wrapped and have its bytes
transcoded as they are written.
// We're wrapping stdout for simplicity, but a File or network reader could

37
converter.go
View File

@ -1,22 +1,23 @@
package iconv
/*
#cgo darwin LDFLAGS: -liconv
#include <stdlib.h>
#include <iconv.h>
*/
import "C"
import "os"
import "syscall"
import "unsafe"
type Converter struct {
context C.iconv_t
open bool
open bool
}
// Initialize a new Converter. If fromEncoding or toEncoding are not supported by
// iconv then an EINVAL error will be returned. An ENOMEM error maybe returned if
// there is not enough memory to initialize an iconv descriptor
func NewConverter(fromEncoding string, toEncoding string) (converter *Converter, err Error) {
func NewConverter(fromEncoding string, toEncoding string) (converter *Converter, err error) {
converter = new(Converter)
// convert to C strings
@ -45,7 +46,7 @@ func (this *Converter) destroy() {
}
// Close a Converter's iconv description explicitly
func (this *Converter) Close() (err os.Error) {
func (this *Converter) Close() (err error) {
if this.open {
_, err = C.iconv_close(this.context)
}
@ -64,7 +65,7 @@ func (this *Converter) Close() (err os.Error) {
// For shift based output encodings, any end shift byte sequences can be generated by
// passing a 0 length byte slice as input. Also passing a 0 length byte slice for output
// will simply reset the iconv descriptor shift state without writing any bytes.
func (this *Converter) Convert(input []byte, output []byte) (bytesRead int, bytesWritten int, err Error) {
func (this *Converter) Convert(input []byte, output []byte) (bytesRead int, bytesWritten int, err error) {
// make sure we are still open
if this.open {
inputLeft := C.size_t(len(input))
@ -77,7 +78,7 @@ func (this *Converter) Convert(input []byte, output []byte) (bytesRead int, byte
inputPointer := (*C.char)(unsafe.Pointer(&input[0]))
outputPointer := (*C.char)(unsafe.Pointer(&output[0]))
_,err = C.iconv(this.context, &inputPointer, &inputLeft, &outputPointer, &outputLeft)
_, err = C.iconv(this.context, &inputPointer, &inputLeft, &outputPointer, &outputLeft)
// update byte counters
bytesRead = len(input) - int(inputLeft)
@ -86,16 +87,16 @@ func (this *Converter) Convert(input []byte, output []byte) (bytesRead int, byte
// inputPointer will be nil, outputPointer is generated as above
outputPointer := (*C.char)(unsafe.Pointer(&output[0]))
_,err = C.iconv(this.context, nil, &inputLeft, &outputPointer, &outputLeft)
_, err = C.iconv(this.context, nil, &inputLeft, &outputPointer, &outputLeft)
// update write byte counter
bytesWritten = len(output) - int(outputLeft)
} else {
// both input and output are zero length, do a shift state reset
_,err = C.iconv(this.context, nil, &inputLeft, nil, &outputLeft)
_, err = C.iconv(this.context, nil, &inputLeft, nil, &outputLeft)
}
} else {
err = EBADF
err = syscall.EBADF
}
return bytesRead, bytesWritten, err
@ -105,12 +106,12 @@ func (this *Converter) Convert(input []byte, output []byte) (bytesRead int, byte
//
// EILSEQ error may be returned if input contains invalid bytes for the
// Converter's fromEncoding.
func (this *Converter) ConvertString(input string) (output string, err Error) {
func (this *Converter) ConvertString(input string) (output string, err error) {
// make sure we are still open
if this.open {
// construct the buffers
inputBuffer := []byte(input)
outputBuffer := make([]byte, len(inputBuffer) * 2) // we use a larger buffer to help avoid resizing later
outputBuffer := make([]byte, len(inputBuffer)*2) // we use a larger buffer to help avoid resizing later
// call Convert until all input bytes are read or an error occurs
var bytesRead, totalBytesRead, bytesWritten, totalBytesWritten int
@ -124,11 +125,11 @@ func (this *Converter) ConvertString(input string) (output string, err Error) {
// check for the E2BIG error specifically, we can add to the output
// buffer to correct for it and then continue
if err == E2BIG {
if err == syscall.E2BIG {
// increase the size of the output buffer by another input length
// first, create a new buffer
tempBuffer := make([]byte, len(outputBuffer) + len(inputBuffer))
tempBuffer := make([]byte, len(outputBuffer)+len(inputBuffer))
// copy the existing data
copy(tempBuffer, outputBuffer)
@ -139,19 +140,19 @@ func (this *Converter) ConvertString(input string) (output string, err Error) {
err = nil
}
}
if err == nil {
// perform a final shift state reset
_, bytesWritten, err = this.Convert([]byte{}, outputBuffer[totalBytesWritten:])
// update total count
totalBytesWritten += bytesWritten
}
// construct the final output string
output = string(outputBuffer[:totalBytesWritten])
} else {
err = EBADF
err = syscall.EBADF
}
return output, err

26
iconv.go
View File

@ -5,26 +5,8 @@ some convenient interface implementations like a Reader and Writer.
*/
package iconv
/*
#include <errno.h>
*/
import "C"
import "os"
// Alias os.Error for convenience
type Error os.Error
// Error codes returned from iconv functions
var (
E2BIG Error = os.Errno(int(C.E2BIG))
EBADF Error = os.Errno(int(C.EBADF))
EINVAL Error = os.Errno(int(C.EINVAL))
EILSEQ Error = os.Errno(int(C.EILSEQ))
ENOMEM Error = os.Errno(int(C.ENOMEM))
)
// All in one Convert method, rather than requiring the construction of an iconv.Converter
func Convert(input []byte, output []byte, fromEncoding string, toEncoding string) (bytesRead int, bytesWritten int, err Error) {
func Convert(input []byte, output []byte, fromEncoding string, toEncoding string) (bytesRead int, bytesWritten int, err error) {
// create a temporary converter
converter, err := NewConverter(fromEncoding, toEncoding)
@ -34,10 +16,10 @@ func Convert(input []byte, output []byte, fromEncoding string, toEncoding string
if err == nil {
var shiftBytesWritten int
// call Convert with a nil input to generate any end shift sequences
_, shiftBytesWritten, err = converter.Convert(nil, output[bytesWritten:])
// add shift bytes to total bytes
bytesWritten += shiftBytesWritten
}
@ -50,7 +32,7 @@ func Convert(input []byte, output []byte, fromEncoding string, toEncoding string
}
// All in one ConvertString method, rather than requiring the construction of an iconv.Converter
func ConvertString(input string, fromEncoding string, toEncoding string) (output string, err Error) {
func ConvertString(input string, fromEncoding string, toEncoding string) (output string, err error) {
// create a temporary converter
converter, err := NewConverter(fromEncoding, toEncoding)

43
iconv_test.go
View File

@ -1,21 +1,22 @@
package iconv
import (
"syscall"
"testing"
)
type iconvTest struct {
description string
input string
inputEncoding string
output string
description string
input string
inputEncoding string
output string
outputEncoding string
bytesRead int
bytesWritten int
err Error
bytesRead int
bytesWritten int
err error
}
var iconvTests = []iconvTest {
var iconvTests = []iconvTest{
iconvTest{
"simple utf-8 to latin1 conversion success",
"Hello World!", "utf-8",
@ -26,25 +27,25 @@ var iconvTests = []iconvTest {
"invalid source encoding causes EINVAL",
"", "doesnotexist",
"", "utf-8",
0, 0, EINVAL,
0, 0, syscall.EINVAL,
},
iconvTest{
"invalid destination encoding causes EINVAL",
"", "utf-8",
"", "doesnotexist",
0, 0, EINVAL,
0, 0, syscall.EINVAL,
},
iconvTest{
"invalid input sequence causes EILSEQ",
"\xFF", "utf-8",
"", "latin1",
0, 0, EILSEQ,
0, 0, syscall.EILSEQ,
},
iconvTest{
"invalid input causes partial output and EILSEQ",
"Hello\xFF", "utf-8",
"Hello", "latin1",
5, 5, EILSEQ,
5, 5, syscall.EILSEQ,
},
}
@ -52,12 +53,12 @@ func TestConvertString(t *testing.T) {
for _, test := range iconvTests {
// perform the conversion
output, err := ConvertString(test.input, test.inputEncoding, test.outputEncoding)
// check that output and err match
if output != test.output {
t.Errorf("test \"%s\" failed, output did not match expected", test.description)
}
// check that err is same as expected
if err != test.err {
if test.err != nil {
@ -77,29 +78,29 @@ func TestConvert(t *testing.T) {
for _, test := range iconvTests {
// setup input buffer
input := []byte(test.input)
// setup a buffer as large as the expected bytesWritten
output := make([]byte, 50)
// peform the conversion
bytesRead, bytesWritten, err := Convert(input, output, test.inputEncoding, test.outputEncoding)
// check that bytesRead is same as expected
if bytesRead != test.bytesRead {
t.Errorf("test \"%s\" failed, bytesRead did not match expected", test.description)
}
// check that bytesWritten is same as expected
if bytesWritten != test.bytesWritten {
t.Errorf("test \"%s\" failed, bytesWritten did not match expected", test.description)
}
// check output bytes against expected - simplest to convert output to
// string and then do an equality check which is actually a byte wise operation
if string(output[:bytesWritten]) != test.output {
t.Errorf("test \"%s\" failed, output did not match expected", test.description)
}
// check that err is same as expected
if err != test.err {
if test.err != nil {
@ -113,4 +114,4 @@ func TestConvert(t *testing.T) {
}
}
}
}
}

22
reader.go
View File

@ -1,19 +1,19 @@
package iconv
import (
import (
"io"
"os"
"syscall"
)
type Reader struct {
source io.Reader
converter *Converter
buffer []byte
source io.Reader
converter *Converter
buffer []byte
readPos, writePos int
err os.Error
err error
}
func NewReader(source io.Reader, fromEncoding string, toEncoding string) (*Reader, os.Error) {
func NewReader(source io.Reader, fromEncoding string, toEncoding string) (*Reader, error) {
// create a converter
converter, err := NewConverter(fromEncoding, toEncoding)
@ -33,7 +33,7 @@ func NewReaderFromConverter(source io.Reader, converter *Converter) (reader *Rea
reader.converter = converter
// create 8K buffers
reader.buffer = make([]byte, 8 * 1024)
reader.buffer = make([]byte, 8*1024)
return reader
}
@ -62,7 +62,7 @@ func (this *Reader) fillBuffer() {
}
// implement the io.Reader interface
func (this *Reader) Read(p []byte) (n int, err os.Error) {
func (this *Reader) Read(p []byte) (n int, err error) {
// checks for when we have no data
for this.writePos == 0 || this.readPos == this.writePos {
// if we have an error / EOF, just return it
@ -72,7 +72,7 @@ func (this *Reader) Read(p []byte) (n int, err os.Error) {
// else, fill our buffer
this.fillBuffer()
}
}
// TODO: checks for when we have less data than len(p)
@ -89,7 +89,7 @@ func (this *Reader) Read(p []byte) (n int, err os.Error) {
// as at least 1 byte was written. If we experienced an E2BIG
// and no bytes were written then the buffer is too small for
// even the next character
if err != E2BIG || bytesWritten == 0 {
if err != syscall.E2BIG || bytesWritten == 0 {
// track anything else
this.err = err
}

33
writer.go
View File

@ -1,19 +1,16 @@
package iconv
import (
"io"
"os"
)
import "io"
type Writer struct {
destination io.Writer
converter *Converter
buffer []byte
destination io.Writer
converter *Converter
buffer []byte
readPos, writePos int
err os.Error
err error
}
func NewWriter(destination io.Writer, fromEncoding string, toEncoding string) (*Writer, os.Error) {
func NewWriter(destination io.Writer, fromEncoding string, toEncoding string) (*Writer, error) {
// create a converter
converter, err := NewConverter(fromEncoding, toEncoding)
@ -33,7 +30,7 @@ func NewWriterFromConverter(destination io.Writer, converter *Converter) (writer
writer.converter = converter
// create 8K buffers
writer.buffer = make([]byte, 8 * 1024)
writer.buffer = make([]byte, 8*1024)
return writer
}
@ -41,10 +38,10 @@ func NewWriterFromConverter(destination io.Writer, converter *Converter) (writer
func (this *Writer) emptyBuffer() {
// write new data out of buffer
bytesWritten, err := this.destination.Write(this.buffer[this.readPos:this.writePos])
// update read position
this.readPos += bytesWritten
// slide existing data to beginning
if this.readPos > 0 {
// copy current bytes - is this guaranteed safe?
@ -62,24 +59,24 @@ func (this *Writer) emptyBuffer() {
}
// implement the io.Writer interface
func (this *Writer) Write(p []byte) (n int, err os.Error) {
func (this *Writer) Write(p []byte) (n int, err error) {
// write data into our internal buffer
bytesRead, bytesWritten, err := this.converter.Convert(p, this.buffer[this.writePos:])
// update bytes written for return
n += bytesRead
this.writePos += bytesWritten
// checks for when we have a full buffer
for this.writePos > 0 {
// if we have an error, just return it
if this.err != nil {
return
return
}
// else empty the buffer
this.emptyBuffer()
}
return n, err
}