go1 updates: standard errors now in syscall package, no more makefile support
This commit is contained in:
parent
8c9fe240c5
commit
d8ca80955a
23
Makefile
23
Makefile
@ -1,23 +0,0 @@
|
|||||||
# standard GO make file preamble
|
|
||||||
include $(GOROOT)/src/Make.inc
|
|
||||||
|
|
||||||
# target package name
|
|
||||||
TARG=iconv
|
|
||||||
|
|
||||||
# regular go files
|
|
||||||
GOFILES=\
|
|
||||||
reader.go\
|
|
||||||
writer.go\
|
|
||||||
|
|
||||||
# files that must be processed by cgo
|
|
||||||
CGOFILES=\
|
|
||||||
converter.go\
|
|
||||||
iconv.go\
|
|
||||||
|
|
||||||
# on non glibc systems, we usually need to load the library
|
|
||||||
ifneq ($(GOOS),linux)
|
|
||||||
CGO_LDFLAGS=-liconv
|
|
||||||
endif
|
|
||||||
|
|
||||||
# standard GO make file include for packages
|
|
||||||
include $(GOROOT)/src/Make.pkg
|
|
26
README.md
26
README.md
@ -1,28 +1,14 @@
|
|||||||
# Install
|
# Install
|
||||||
|
|
||||||
The main method of installation is through gomake (provided in $GOROOT/bin)
|
The main method of installation is through "go install" (provided in $GOROOT/bin)
|
||||||
|
|
||||||
git clone git://github.com/djimenez/iconv.go.git iconv
|
go install github.com/djimenez/iconv.go
|
||||||
cd iconv
|
|
||||||
gomake install
|
|
||||||
|
|
||||||
Alternatively, you can try using goinstall (also provided in $GOROOT/bin).
|
|
||||||
However, because iconv.go uses cgo to wrap iconv functions, the build may not
|
|
||||||
succeed on all systems. At time of writing goinstall was still experimental and
|
|
||||||
has known issues with cgo based packages because of how it produces its own
|
|
||||||
make file.
|
|
||||||
|
|
||||||
goinstall github.com/djimenez/iconv.go
|
|
||||||
|
|
||||||
# Usage
|
# Usage
|
||||||
|
|
||||||
To use the package, you'll need the appropriate import statement:
|
To use the package, you'll need the appropriate import statement:
|
||||||
|
|
||||||
import (
|
import (
|
||||||
// if you used gomake install directly, you'll want this import
|
|
||||||
iconv
|
|
||||||
|
|
||||||
// if you used goinstall, you'll want this import
|
|
||||||
iconv "github.com/djimenez/iconv.go"
|
iconv "github.com/djimenez/iconv.go"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -96,18 +82,18 @@ automatically do this since it can be used to process a full stream in chunks.
|
|||||||
So you'll need to remember to pass a nil input buffer at the end yourself, just
|
So you'll need to remember to pass a nil input buffer at the end yourself, just
|
||||||
like you would with direct iconv usage.
|
like you would with direct iconv usage.
|
||||||
|
|
||||||
## Converting an *io.Reader
|
## Converting an \*io.Reader
|
||||||
|
|
||||||
The iconv.Reader allows any other *io.Reader to be wrapped and have its bytes
|
The iconv.Reader allows any other \*io.Reader to be wrapped and have its bytes
|
||||||
transcoded as they are read.
|
transcoded as they are read.
|
||||||
|
|
||||||
// We're wrapping stdin for simplicity, but a File or network reader could
|
// We're wrapping stdin for simplicity, but a File or network reader could
|
||||||
// be wrapped as well
|
// be wrapped as well
|
||||||
reader,_ := iconv.NewReader(os.Stdin, "utf-8", "windows-1252")
|
reader,_ := iconv.NewReader(os.Stdin, "utf-8", "windows-1252")
|
||||||
|
|
||||||
## Converting an *io.Writer
|
## Converting an \*io.Writer
|
||||||
|
|
||||||
The iconv.Writer allows any other *io.Writer to be wrapped and have its bytes
|
The iconv.Writer allows any other \*io.Writer to be wrapped and have its bytes
|
||||||
transcoded as they are written.
|
transcoded as they are written.
|
||||||
|
|
||||||
// We're wrapping stdout for simplicity, but a File or network reader could
|
// We're wrapping stdout for simplicity, but a File or network reader could
|
||||||
|
27
converter.go
27
converter.go
@ -1,11 +1,12 @@
|
|||||||
package iconv
|
package iconv
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
#cgo darwin LDFLAGS: -liconv
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <iconv.h>
|
#include <iconv.h>
|
||||||
*/
|
*/
|
||||||
import "C"
|
import "C"
|
||||||
import "os"
|
import "syscall"
|
||||||
import "unsafe"
|
import "unsafe"
|
||||||
|
|
||||||
type Converter struct {
|
type Converter struct {
|
||||||
@ -16,7 +17,7 @@ type Converter struct {
|
|||||||
// Initialize a new Converter. If fromEncoding or toEncoding are not supported by
|
// Initialize a new Converter. If fromEncoding or toEncoding are not supported by
|
||||||
// iconv then an EINVAL error will be returned. An ENOMEM error maybe returned if
|
// iconv then an EINVAL error will be returned. An ENOMEM error maybe returned if
|
||||||
// there is not enough memory to initialize an iconv descriptor
|
// there is not enough memory to initialize an iconv descriptor
|
||||||
func NewConverter(fromEncoding string, toEncoding string) (converter *Converter, err Error) {
|
func NewConverter(fromEncoding string, toEncoding string) (converter *Converter, err error) {
|
||||||
converter = new(Converter)
|
converter = new(Converter)
|
||||||
|
|
||||||
// convert to C strings
|
// convert to C strings
|
||||||
@ -45,7 +46,7 @@ func (this *Converter) destroy() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Close a Converter's iconv description explicitly
|
// Close a Converter's iconv description explicitly
|
||||||
func (this *Converter) Close() (err os.Error) {
|
func (this *Converter) Close() (err error) {
|
||||||
if this.open {
|
if this.open {
|
||||||
_, err = C.iconv_close(this.context)
|
_, err = C.iconv_close(this.context)
|
||||||
}
|
}
|
||||||
@ -64,7 +65,7 @@ func (this *Converter) Close() (err os.Error) {
|
|||||||
// For shift based output encodings, any end shift byte sequences can be generated by
|
// For shift based output encodings, any end shift byte sequences can be generated by
|
||||||
// passing a 0 length byte slice as input. Also passing a 0 length byte slice for output
|
// passing a 0 length byte slice as input. Also passing a 0 length byte slice for output
|
||||||
// will simply reset the iconv descriptor shift state without writing any bytes.
|
// will simply reset the iconv descriptor shift state without writing any bytes.
|
||||||
func (this *Converter) Convert(input []byte, output []byte) (bytesRead int, bytesWritten int, err Error) {
|
func (this *Converter) Convert(input []byte, output []byte) (bytesRead int, bytesWritten int, err error) {
|
||||||
// make sure we are still open
|
// make sure we are still open
|
||||||
if this.open {
|
if this.open {
|
||||||
inputLeft := C.size_t(len(input))
|
inputLeft := C.size_t(len(input))
|
||||||
@ -77,7 +78,7 @@ func (this *Converter) Convert(input []byte, output []byte) (bytesRead int, byte
|
|||||||
inputPointer := (*C.char)(unsafe.Pointer(&input[0]))
|
inputPointer := (*C.char)(unsafe.Pointer(&input[0]))
|
||||||
outputPointer := (*C.char)(unsafe.Pointer(&output[0]))
|
outputPointer := (*C.char)(unsafe.Pointer(&output[0]))
|
||||||
|
|
||||||
_,err = C.iconv(this.context, &inputPointer, &inputLeft, &outputPointer, &outputLeft)
|
_, err = C.iconv(this.context, &inputPointer, &inputLeft, &outputPointer, &outputLeft)
|
||||||
|
|
||||||
// update byte counters
|
// update byte counters
|
||||||
bytesRead = len(input) - int(inputLeft)
|
bytesRead = len(input) - int(inputLeft)
|
||||||
@ -86,16 +87,16 @@ func (this *Converter) Convert(input []byte, output []byte) (bytesRead int, byte
|
|||||||
// inputPointer will be nil, outputPointer is generated as above
|
// inputPointer will be nil, outputPointer is generated as above
|
||||||
outputPointer := (*C.char)(unsafe.Pointer(&output[0]))
|
outputPointer := (*C.char)(unsafe.Pointer(&output[0]))
|
||||||
|
|
||||||
_,err = C.iconv(this.context, nil, &inputLeft, &outputPointer, &outputLeft)
|
_, err = C.iconv(this.context, nil, &inputLeft, &outputPointer, &outputLeft)
|
||||||
|
|
||||||
// update write byte counter
|
// update write byte counter
|
||||||
bytesWritten = len(output) - int(outputLeft)
|
bytesWritten = len(output) - int(outputLeft)
|
||||||
} else {
|
} else {
|
||||||
// both input and output are zero length, do a shift state reset
|
// both input and output are zero length, do a shift state reset
|
||||||
_,err = C.iconv(this.context, nil, &inputLeft, nil, &outputLeft)
|
_, err = C.iconv(this.context, nil, &inputLeft, nil, &outputLeft)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
err = EBADF
|
err = syscall.EBADF
|
||||||
}
|
}
|
||||||
|
|
||||||
return bytesRead, bytesWritten, err
|
return bytesRead, bytesWritten, err
|
||||||
@ -105,12 +106,12 @@ func (this *Converter) Convert(input []byte, output []byte) (bytesRead int, byte
|
|||||||
//
|
//
|
||||||
// EILSEQ error may be returned if input contains invalid bytes for the
|
// EILSEQ error may be returned if input contains invalid bytes for the
|
||||||
// Converter's fromEncoding.
|
// Converter's fromEncoding.
|
||||||
func (this *Converter) ConvertString(input string) (output string, err Error) {
|
func (this *Converter) ConvertString(input string) (output string, err error) {
|
||||||
// make sure we are still open
|
// make sure we are still open
|
||||||
if this.open {
|
if this.open {
|
||||||
// construct the buffers
|
// construct the buffers
|
||||||
inputBuffer := []byte(input)
|
inputBuffer := []byte(input)
|
||||||
outputBuffer := make([]byte, len(inputBuffer) * 2) // we use a larger buffer to help avoid resizing later
|
outputBuffer := make([]byte, len(inputBuffer)*2) // we use a larger buffer to help avoid resizing later
|
||||||
|
|
||||||
// call Convert until all input bytes are read or an error occurs
|
// call Convert until all input bytes are read or an error occurs
|
||||||
var bytesRead, totalBytesRead, bytesWritten, totalBytesWritten int
|
var bytesRead, totalBytesRead, bytesWritten, totalBytesWritten int
|
||||||
@ -124,10 +125,10 @@ func (this *Converter) ConvertString(input string) (output string, err Error) {
|
|||||||
|
|
||||||
// check for the E2BIG error specifically, we can add to the output
|
// check for the E2BIG error specifically, we can add to the output
|
||||||
// buffer to correct for it and then continue
|
// buffer to correct for it and then continue
|
||||||
if err == E2BIG {
|
if err == syscall.E2BIG {
|
||||||
// increase the size of the output buffer by another input length
|
// increase the size of the output buffer by another input length
|
||||||
// first, create a new buffer
|
// first, create a new buffer
|
||||||
tempBuffer := make([]byte, len(outputBuffer) + len(inputBuffer))
|
tempBuffer := make([]byte, len(outputBuffer)+len(inputBuffer))
|
||||||
|
|
||||||
// copy the existing data
|
// copy the existing data
|
||||||
copy(tempBuffer, outputBuffer)
|
copy(tempBuffer, outputBuffer)
|
||||||
@ -151,7 +152,7 @@ func (this *Converter) ConvertString(input string) (output string, err Error) {
|
|||||||
// construct the final output string
|
// construct the final output string
|
||||||
output = string(outputBuffer[:totalBytesWritten])
|
output = string(outputBuffer[:totalBytesWritten])
|
||||||
} else {
|
} else {
|
||||||
err = EBADF
|
err = syscall.EBADF
|
||||||
}
|
}
|
||||||
|
|
||||||
return output, err
|
return output, err
|
||||||
|
22
iconv.go
22
iconv.go
@ -5,26 +5,8 @@ some convenient interface implementations like a Reader and Writer.
|
|||||||
*/
|
*/
|
||||||
package iconv
|
package iconv
|
||||||
|
|
||||||
/*
|
|
||||||
#include <errno.h>
|
|
||||||
*/
|
|
||||||
import "C"
|
|
||||||
import "os"
|
|
||||||
|
|
||||||
// Alias os.Error for convenience
|
|
||||||
type Error os.Error
|
|
||||||
|
|
||||||
// Error codes returned from iconv functions
|
|
||||||
var (
|
|
||||||
E2BIG Error = os.Errno(int(C.E2BIG))
|
|
||||||
EBADF Error = os.Errno(int(C.EBADF))
|
|
||||||
EINVAL Error = os.Errno(int(C.EINVAL))
|
|
||||||
EILSEQ Error = os.Errno(int(C.EILSEQ))
|
|
||||||
ENOMEM Error = os.Errno(int(C.ENOMEM))
|
|
||||||
)
|
|
||||||
|
|
||||||
// All in one Convert method, rather than requiring the construction of an iconv.Converter
|
// All in one Convert method, rather than requiring the construction of an iconv.Converter
|
||||||
func Convert(input []byte, output []byte, fromEncoding string, toEncoding string) (bytesRead int, bytesWritten int, err Error) {
|
func Convert(input []byte, output []byte, fromEncoding string, toEncoding string) (bytesRead int, bytesWritten int, err error) {
|
||||||
// create a temporary converter
|
// create a temporary converter
|
||||||
converter, err := NewConverter(fromEncoding, toEncoding)
|
converter, err := NewConverter(fromEncoding, toEncoding)
|
||||||
|
|
||||||
@ -50,7 +32,7 @@ func Convert(input []byte, output []byte, fromEncoding string, toEncoding string
|
|||||||
}
|
}
|
||||||
|
|
||||||
// All in one ConvertString method, rather than requiring the construction of an iconv.Converter
|
// All in one ConvertString method, rather than requiring the construction of an iconv.Converter
|
||||||
func ConvertString(input string, fromEncoding string, toEncoding string) (output string, err Error) {
|
func ConvertString(input string, fromEncoding string, toEncoding string) (output string, err error) {
|
||||||
// create a temporary converter
|
// create a temporary converter
|
||||||
converter, err := NewConverter(fromEncoding, toEncoding)
|
converter, err := NewConverter(fromEncoding, toEncoding)
|
||||||
|
|
||||||
|
@ -1,6 +1,7 @@
|
|||||||
package iconv
|
package iconv
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"syscall"
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -12,10 +13,10 @@ type iconvTest struct {
|
|||||||
outputEncoding string
|
outputEncoding string
|
||||||
bytesRead int
|
bytesRead int
|
||||||
bytesWritten int
|
bytesWritten int
|
||||||
err Error
|
err error
|
||||||
}
|
}
|
||||||
|
|
||||||
var iconvTests = []iconvTest {
|
var iconvTests = []iconvTest{
|
||||||
iconvTest{
|
iconvTest{
|
||||||
"simple utf-8 to latin1 conversion success",
|
"simple utf-8 to latin1 conversion success",
|
||||||
"Hello World!", "utf-8",
|
"Hello World!", "utf-8",
|
||||||
@ -26,25 +27,25 @@ var iconvTests = []iconvTest {
|
|||||||
"invalid source encoding causes EINVAL",
|
"invalid source encoding causes EINVAL",
|
||||||
"", "doesnotexist",
|
"", "doesnotexist",
|
||||||
"", "utf-8",
|
"", "utf-8",
|
||||||
0, 0, EINVAL,
|
0, 0, syscall.EINVAL,
|
||||||
},
|
},
|
||||||
iconvTest{
|
iconvTest{
|
||||||
"invalid destination encoding causes EINVAL",
|
"invalid destination encoding causes EINVAL",
|
||||||
"", "utf-8",
|
"", "utf-8",
|
||||||
"", "doesnotexist",
|
"", "doesnotexist",
|
||||||
0, 0, EINVAL,
|
0, 0, syscall.EINVAL,
|
||||||
},
|
},
|
||||||
iconvTest{
|
iconvTest{
|
||||||
"invalid input sequence causes EILSEQ",
|
"invalid input sequence causes EILSEQ",
|
||||||
"\xFF", "utf-8",
|
"\xFF", "utf-8",
|
||||||
"", "latin1",
|
"", "latin1",
|
||||||
0, 0, EILSEQ,
|
0, 0, syscall.EILSEQ,
|
||||||
},
|
},
|
||||||
iconvTest{
|
iconvTest{
|
||||||
"invalid input causes partial output and EILSEQ",
|
"invalid input causes partial output and EILSEQ",
|
||||||
"Hello\xFF", "utf-8",
|
"Hello\xFF", "utf-8",
|
||||||
"Hello", "latin1",
|
"Hello", "latin1",
|
||||||
5, 5, EILSEQ,
|
5, 5, syscall.EILSEQ,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
12
reader.go
12
reader.go
@ -2,7 +2,7 @@ package iconv
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"syscall"
|
||||||
)
|
)
|
||||||
|
|
||||||
type Reader struct {
|
type Reader struct {
|
||||||
@ -10,10 +10,10 @@ type Reader struct {
|
|||||||
converter *Converter
|
converter *Converter
|
||||||
buffer []byte
|
buffer []byte
|
||||||
readPos, writePos int
|
readPos, writePos int
|
||||||
err os.Error
|
err error
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewReader(source io.Reader, fromEncoding string, toEncoding string) (*Reader, os.Error) {
|
func NewReader(source io.Reader, fromEncoding string, toEncoding string) (*Reader, error) {
|
||||||
// create a converter
|
// create a converter
|
||||||
converter, err := NewConverter(fromEncoding, toEncoding)
|
converter, err := NewConverter(fromEncoding, toEncoding)
|
||||||
|
|
||||||
@ -33,7 +33,7 @@ func NewReaderFromConverter(source io.Reader, converter *Converter) (reader *Rea
|
|||||||
reader.converter = converter
|
reader.converter = converter
|
||||||
|
|
||||||
// create 8K buffers
|
// create 8K buffers
|
||||||
reader.buffer = make([]byte, 8 * 1024)
|
reader.buffer = make([]byte, 8*1024)
|
||||||
|
|
||||||
return reader
|
return reader
|
||||||
}
|
}
|
||||||
@ -62,7 +62,7 @@ func (this *Reader) fillBuffer() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// implement the io.Reader interface
|
// implement the io.Reader interface
|
||||||
func (this *Reader) Read(p []byte) (n int, err os.Error) {
|
func (this *Reader) Read(p []byte) (n int, err error) {
|
||||||
// checks for when we have no data
|
// checks for when we have no data
|
||||||
for this.writePos == 0 || this.readPos == this.writePos {
|
for this.writePos == 0 || this.readPos == this.writePos {
|
||||||
// if we have an error / EOF, just return it
|
// if we have an error / EOF, just return it
|
||||||
@ -89,7 +89,7 @@ func (this *Reader) Read(p []byte) (n int, err os.Error) {
|
|||||||
// as at least 1 byte was written. If we experienced an E2BIG
|
// as at least 1 byte was written. If we experienced an E2BIG
|
||||||
// and no bytes were written then the buffer is too small for
|
// and no bytes were written then the buffer is too small for
|
||||||
// even the next character
|
// even the next character
|
||||||
if err != E2BIG || bytesWritten == 0 {
|
if err != syscall.E2BIG || bytesWritten == 0 {
|
||||||
// track anything else
|
// track anything else
|
||||||
this.err = err
|
this.err = err
|
||||||
}
|
}
|
||||||
|
13
writer.go
13
writer.go
@ -1,19 +1,16 @@
|
|||||||
package iconv
|
package iconv
|
||||||
|
|
||||||
import (
|
import "io"
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
)
|
|
||||||
|
|
||||||
type Writer struct {
|
type Writer struct {
|
||||||
destination io.Writer
|
destination io.Writer
|
||||||
converter *Converter
|
converter *Converter
|
||||||
buffer []byte
|
buffer []byte
|
||||||
readPos, writePos int
|
readPos, writePos int
|
||||||
err os.Error
|
err error
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewWriter(destination io.Writer, fromEncoding string, toEncoding string) (*Writer, os.Error) {
|
func NewWriter(destination io.Writer, fromEncoding string, toEncoding string) (*Writer, error) {
|
||||||
// create a converter
|
// create a converter
|
||||||
converter, err := NewConverter(fromEncoding, toEncoding)
|
converter, err := NewConverter(fromEncoding, toEncoding)
|
||||||
|
|
||||||
@ -33,7 +30,7 @@ func NewWriterFromConverter(destination io.Writer, converter *Converter) (writer
|
|||||||
writer.converter = converter
|
writer.converter = converter
|
||||||
|
|
||||||
// create 8K buffers
|
// create 8K buffers
|
||||||
writer.buffer = make([]byte, 8 * 1024)
|
writer.buffer = make([]byte, 8*1024)
|
||||||
|
|
||||||
return writer
|
return writer
|
||||||
}
|
}
|
||||||
@ -62,7 +59,7 @@ func (this *Writer) emptyBuffer() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// implement the io.Writer interface
|
// implement the io.Writer interface
|
||||||
func (this *Writer) Write(p []byte) (n int, err os.Error) {
|
func (this *Writer) Write(p []byte) (n int, err error) {
|
||||||
// write data into our internal buffer
|
// write data into our internal buffer
|
||||||
bytesRead, bytesWritten, err := this.converter.Convert(p, this.buffer[this.writePos:])
|
bytesRead, bytesWritten, err := this.converter.Convert(p, this.buffer[this.writePos:])
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user