package parse import ( "encoding/binary" "errors" "fmt" "io" "math" "os" ) const PageSize = 4096 // BinaryReader is a binary big endian file format reader. type BinaryReader struct { Endianness binary.ByteOrder buf []byte pos uint32 eof bool } // NewBinaryReader returns a big endian binary file format reader. func NewBinaryReader(buf []byte) *BinaryReader { if math.MaxUint32 < uint(len(buf)) { return &BinaryReader{binary.BigEndian, nil, 0, true} } return &BinaryReader{binary.BigEndian, buf, 0, false} } // NewBinaryReaderLE returns a little endian binary file format reader. func NewBinaryReaderLE(buf []byte) *BinaryReader { r := NewBinaryReader(buf) r.Endianness = binary.LittleEndian return r } // Seek set the reader position in the buffer. func (r *BinaryReader) Seek(pos uint32) error { if uint32(len(r.buf)) < pos { r.eof = true return io.EOF } r.pos = pos r.eof = false return nil } // Pos returns the reader's position. func (r *BinaryReader) Pos() uint32 { return r.pos } // Len returns the remaining length of the buffer. func (r *BinaryReader) Len() uint32 { return uint32(len(r.buf)) - r.pos } // EOF returns true if we reached the end-of-file. func (r *BinaryReader) EOF() bool { return r.eof } // Read complies with io.Reader. func (r *BinaryReader) Read(b []byte) (int, error) { n := copy(b, r.buf[r.pos:]) r.pos += uint32(n) if r.pos == uint32(len(r.buf)) { r.eof = true return n, io.EOF } return n, nil } // ReadBytes reads n bytes. func (r *BinaryReader) ReadBytes(n uint32) []byte { if r.eof || uint32(len(r.buf))-r.pos < n { r.eof = true return nil } buf := r.buf[r.pos : r.pos+n : r.pos+n] r.pos += n return buf } // ReadString reads a string of length n. func (r *BinaryReader) ReadString(n uint32) string { return string(r.ReadBytes(n)) } // ReadByte reads a single byte. func (r *BinaryReader) ReadByte() byte { b := r.ReadBytes(1) if b == nil { return 0 } return b[0] } // ReadUint8 reads a uint8. func (r *BinaryReader) ReadUint8() uint8 { return r.ReadByte() } // ReadUint16 reads a uint16. func (r *BinaryReader) ReadUint16() uint16 { b := r.ReadBytes(2) if b == nil { return 0 } return r.Endianness.Uint16(b) } // ReadUint32 reads a uint32. func (r *BinaryReader) ReadUint32() uint32 { b := r.ReadBytes(4) if b == nil { return 0 } return r.Endianness.Uint32(b) } // ReadUint64 reads a uint64. func (r *BinaryReader) ReadUint64() uint64 { b := r.ReadBytes(8) if b == nil { return 0 } return r.Endianness.Uint64(b) } // ReadInt8 reads a int8. func (r *BinaryReader) ReadInt8() int8 { return int8(r.ReadByte()) } // ReadInt16 reads a int16. func (r *BinaryReader) ReadInt16() int16 { return int16(r.ReadUint16()) } // ReadInt32 reads a int32. func (r *BinaryReader) ReadInt32() int32 { return int32(r.ReadUint32()) } // ReadInt64 reads a int64. func (r *BinaryReader) ReadInt64() int64 { return int64(r.ReadUint64()) } type BinaryFileReader struct { f *os.File size uint64 offset uint64 Endianness binary.ByteOrder buf []byte pos int } func NewBinaryFileReader(f *os.File, chunk int) (*BinaryFileReader, error) { var buf []byte var size uint64 if chunk == 0 { var err error if buf, err = io.ReadAll(f); err != nil { return nil, err } } else { buf = make([]byte, 0, chunk) } if info, err := f.Stat(); err != nil { return nil, err } else { size = uint64(info.Size()) } return &BinaryFileReader{ f: f, size: size, Endianness: binary.BigEndian, buf: buf, }, nil } func (r *BinaryFileReader) buffer(pos, length uint64) error { if pos < r.offset || r.offset+uint64(len(r.buf)) < pos+length { if math.MaxInt64 < pos { return fmt.Errorf("seek position too large") } else if _, err := r.f.Seek(int64(pos), 0); err != nil { return err } else if n, err := r.f.Read(r.buf[:cap(r.buf)]); err != nil { return err } else { r.offset = pos r.buf = r.buf[:n] r.pos = 0 } } return nil } // Seek set the reader position in the buffer. func (r *BinaryFileReader) Seek(pos uint64) error { if r.size <= pos { return io.EOF } else if err := r.buffer(pos, 0); err != nil { return err } r.pos = int(pos - r.offset) return nil } // Pos returns the reader's position. func (r *BinaryFileReader) Pos() uint64 { return r.offset + uint64(r.pos) } // Len returns the remaining length of the buffer. func (r *BinaryFileReader) Len() uint64 { return r.size - r.Pos() } // Offset returns the offset of the buffer. func (r *BinaryFileReader) Offset() uint64 { return r.offset } // BufferLen returns the length of the buffer. func (r *BinaryFileReader) BufferLen() int { return len(r.buf) } // Read complies with io.Reader. func (r *BinaryFileReader) Read(b []byte) (int, error) { if len(b) <= cap(r.buf) { if err := r.buffer(r.offset+uint64(r.pos), uint64(len(b))); err != nil { return 0, err } n := copy(b, r.buf[r.pos:]) r.pos += n return n, nil } // read directly from file if _, err := r.f.Seek(int64(r.offset)+int64(r.pos), 0); err != nil { return 0, err } n, err := r.f.Read(b) r.offset += uint64(r.pos + n) r.pos = 0 r.buf = r.buf[:0] return n, err } // ReadBytes reads n bytes. func (r *BinaryFileReader) ReadBytes(n int) []byte { if n < len(r.buf)-r.pos { b := r.buf[r.pos : r.pos+n] r.pos += n return b } b := make([]byte, n) if _, err := r.Read(b); err != nil { return nil } return b } // ReadString reads a string of length n. func (r *BinaryFileReader) ReadString(n int) string { return string(r.ReadBytes(n)) } // ReadByte reads a single byte. func (r *BinaryFileReader) ReadByte() byte { b := r.ReadBytes(1) if b == nil { return 0 } return b[0] } // ReadUint8 reads a uint8. func (r *BinaryFileReader) ReadUint8() uint8 { return r.ReadByte() } // ReadUint16 reads a uint16. func (r *BinaryFileReader) ReadUint16() uint16 { b := r.ReadBytes(2) if b == nil { return 0 } return r.Endianness.Uint16(b) } // ReadUint32 reads a uint32. func (r *BinaryFileReader) ReadUint32() uint32 { b := r.ReadBytes(4) if b == nil { return 0 } return r.Endianness.Uint32(b) } // ReadUint64 reads a uint64. func (r *BinaryFileReader) ReadUint64() uint64 { b := r.ReadBytes(8) if b == nil { return 0 } return r.Endianness.Uint64(b) } // ReadInt8 reads a int8. func (r *BinaryFileReader) ReadInt8() int8 { return int8(r.ReadByte()) } // ReadInt16 reads a int16. func (r *BinaryFileReader) ReadInt16() int16 { return int16(r.ReadUint16()) } // ReadInt32 reads a int32. func (r *BinaryFileReader) ReadInt32() int32 { return int32(r.ReadUint32()) } // ReadInt64 reads a int64. func (r *BinaryFileReader) ReadInt64() int64 { return int64(r.ReadUint64()) } type IBinaryReader interface { Close() error Len() int Bytes(int, int64) ([]byte, error) } type binaryReaderFile struct { f *os.File size int64 } func newBinaryReaderFile(filename string) (*binaryReaderFile, error) { f, err := os.Open(filename) if err != nil { return nil, err } fi, err := f.Stat() if err != nil { f.Close() return nil, err } return &binaryReaderFile{f, fi.Size()}, nil } // Close closes the reader. func (r *binaryReaderFile) Close() error { return r.f.Close() } // Len returns the length of the underlying memory-mapped file. func (r *binaryReaderFile) Len() int { return int(r.size) } func (r *binaryReaderFile) Bytes(n int, off int64) ([]byte, error) { if _, err := r.f.Seek(off, 0); err != nil { return nil, err } b := make([]byte, n) m, err := r.f.Read(b) if err != nil { return nil, err } else if m != n { return nil, errors.New("file: could not read all bytes") } return b, nil } type binaryReaderBytes struct { data []byte } func newBinaryReaderBytes(data []byte) (*binaryReaderBytes, error) { return &binaryReaderBytes{data}, nil } // Close closes the reader. func (r *binaryReaderBytes) Close() error { return nil } // Len returns the length of the underlying memory-mapped file. func (r *binaryReaderBytes) Len() int { return len(r.data) } func (r *binaryReaderBytes) Bytes(n int, off int64) ([]byte, error) { if off < 0 || int64(len(r.data)) < off { return nil, fmt.Errorf("bytes: invalid offset %d", off) } return r.data[off : off+int64(n) : off+int64(n)], nil } type binaryReaderReader struct { r io.Reader n int64 readerAt bool seeker bool } func newBinaryReaderReader(r io.Reader, n int64) (*binaryReaderReader, error) { _, readerAt := r.(io.ReaderAt) _, seeker := r.(io.Seeker) return &binaryReaderReader{r, n, readerAt, seeker}, nil } // Close closes the reader. func (r *binaryReaderReader) Close() error { if closer, ok := r.r.(io.Closer); ok { return closer.Close() } return nil } // Len returns the length of the underlying memory-mapped file. func (r *binaryReaderReader) Len() int { return int(r.n) } func (r *binaryReaderReader) Bytes(n int, off int64) ([]byte, error) { // seeker seems faster than readerAt by 10% if r.seeker { if _, err := r.r.(io.Seeker).Seek(off, 0); err != nil { return nil, err } b := make([]byte, n) m, err := r.r.Read(b) if err != nil { return nil, err } else if m != n { return nil, errors.New("file: could not read all bytes") } return b, nil } else if r.readerAt { b := make([]byte, n) m, err := r.r.(io.ReaderAt).ReadAt(b, off) if err != nil { return nil, err } else if m != n { return nil, errors.New("file: could not read all bytes") } return b, nil } return nil, errors.New("io.Seeker and io.ReaderAt not implemented") } type BinaryReader2 struct { f IBinaryReader pos int64 err error Endian binary.ByteOrder } func NewBinaryReader2(f IBinaryReader) *BinaryReader2 { return &BinaryReader2{ f: f, Endian: binary.BigEndian, } } func NewBinaryReader2Reader(r io.Reader, n int64) (*BinaryReader2, error) { _, isReaderAt := r.(io.ReaderAt) _, isSeeker := r.(io.Seeker) var f IBinaryReader if isReaderAt || isSeeker { var err error f, err = newBinaryReaderReader(r, n) if err != nil { return nil, err } } else { b := make([]byte, n) if _, err := io.ReadFull(r, b); err != nil { return nil, err } f, _ = newBinaryReaderBytes(b) } return NewBinaryReader2(f), nil } func NewBinaryReader2Bytes(data []byte) (*BinaryReader2, error) { f, _ := newBinaryReaderBytes(data) return NewBinaryReader2(f), nil } func NewBinaryReader2File(filename string) (*BinaryReader2, error) { f, err := newBinaryReaderFile(filename) if err != nil { return nil, err } return NewBinaryReader2(f), nil } func NewBinaryReader2Mmap(filename string) (*BinaryReader2, error) { f, err := newBinaryReaderMmap(filename) if err != nil { return nil, err } return NewBinaryReader2(f), nil } func (r *BinaryReader2) Err() error { return r.err } func (r *BinaryReader2) Close() error { if err := r.f.Close(); err != nil { return err } return r.err } // InPageCache returns true if the range is already in the page cache (for mmap). func (r *BinaryReader2) InPageCache(start, end int64) bool { index := int64(r.Pos()) / PageSize return start/PageSize == index && end/PageSize == index } // Free frees all previously read bytes, you cannot seek from before this position (for reader). func (r *BinaryReader2) Free() { } // Pos returns the reader's position. func (r *BinaryReader2) Pos() int64 { return r.pos } // Len returns the remaining length of the buffer. func (r *BinaryReader2) Len() int { return int(int64(r.f.Len()) - int64(r.pos)) } func (r *BinaryReader2) Seek(pos int64) { r.pos = pos } // Read complies with io.Reader. func (r *BinaryReader2) Read(b []byte) (int, error) { data, err := r.f.Bytes(len(b), r.pos) if err != nil && err != io.EOF { return 0, err } n := copy(b, data) r.pos += int64(len(b)) return n, err } // ReadBytes reads n bytes. func (r *BinaryReader2) ReadBytes(n int) []byte { data, err := r.f.Bytes(n, r.pos) if err != nil { r.err = err return nil } r.pos += int64(n) return data } // ReadString reads a string of length n. func (r *BinaryReader2) ReadString(n int) string { return string(r.ReadBytes(n)) } // ReadByte reads a single byte. func (r *BinaryReader2) ReadByte() byte { data := r.ReadBytes(1) if data == nil { return 0 } return data[0] } // ReadUint8 reads a uint8. func (r *BinaryReader2) ReadUint8() uint8 { return r.ReadByte() } // ReadUint16 reads a uint16. func (r *BinaryReader2) ReadUint16() uint16 { data := r.ReadBytes(2) if data == nil { return 0 } else if r.Endian == binary.LittleEndian { return uint16(data[1])<<8 | uint16(data[0]) } return uint16(data[0])<<8 | uint16(data[1]) } // ReadUint32 reads a uint32. func (r *BinaryReader2) ReadUint32() uint32 { data := r.ReadBytes(4) if data == nil { return 0 } else if r.Endian == binary.LittleEndian { return uint32(data[3])<<24 | uint32(data[2])<<16 | uint32(data[1])<<8 | uint32(data[0]) } return uint32(data[0])<<24 | uint32(data[1])<<16 | uint32(data[2])<<8 | uint32(data[3]) } // ReadUint64 reads a uint64. func (r *BinaryReader2) ReadUint64() uint64 { data := r.ReadBytes(8) if data == nil { return 0 } else if r.Endian == binary.LittleEndian { return uint64(data[7])<<56 | uint64(data[6])<<48 | uint64(data[5])<<40 | uint64(data[4])<<32 | uint64(data[3])<<24 | uint64(data[2])<<16 | uint64(data[1])<<8 | uint64(data[0]) } return uint64(data[0])<<56 | uint64(data[1])<<48 | uint64(data[2])<<40 | uint64(data[3])<<32 | uint64(data[4])<<24 | uint64(data[5])<<16 | uint64(data[6])<<8 | uint64(data[7]) } // ReadInt8 reads a int8. func (r *BinaryReader2) ReadInt8() int8 { return int8(r.ReadByte()) } // ReadInt16 reads a int16. func (r *BinaryReader2) ReadInt16() int16 { return int16(r.ReadUint16()) } // ReadInt32 reads a int32. func (r *BinaryReader2) ReadInt32() int32 { return int32(r.ReadUint32()) } // ReadInt64 reads a int64. func (r *BinaryReader2) ReadInt64() int64 { return int64(r.ReadUint64()) } // BinaryWriter is a big endian binary file format writer. type BinaryWriter struct { buf []byte } // NewBinaryWriter returns a big endian binary file format writer. func NewBinaryWriter(buf []byte) *BinaryWriter { return &BinaryWriter{buf} } // Len returns the buffer's length in bytes. func (w *BinaryWriter) Len() uint32 { return uint32(len(w.buf)) } // Bytes returns the buffer's bytes. func (w *BinaryWriter) Bytes() []byte { return w.buf } // Write complies with io.Writer. func (w *BinaryWriter) Write(b []byte) (int, error) { w.buf = append(w.buf, b...) return len(b), nil } // WriteBytes writes the given bytes to the buffer. func (w *BinaryWriter) WriteBytes(v []byte) { w.buf = append(w.buf, v...) } // WriteString writes the given string to the buffer. func (w *BinaryWriter) WriteString(v string) { w.WriteBytes([]byte(v)) } // WriteByte writes the given byte to the buffer. func (w *BinaryWriter) WriteByte(v byte) { w.WriteBytes([]byte{v}) } // WriteUint8 writes the given uint8 to the buffer. func (w *BinaryWriter) WriteUint8(v uint8) { w.WriteByte(v) } // WriteUint16 writes the given uint16 to the buffer. func (w *BinaryWriter) WriteUint16(v uint16) { pos := len(w.buf) w.buf = append(w.buf, make([]byte, 2)...) binary.BigEndian.PutUint16(w.buf[pos:], v) } // WriteUint32 writes the given uint32 to the buffer. func (w *BinaryWriter) WriteUint32(v uint32) { pos := len(w.buf) w.buf = append(w.buf, make([]byte, 4)...) binary.BigEndian.PutUint32(w.buf[pos:], v) } // WriteUint64 writes the given uint64 to the buffer. func (w *BinaryWriter) WriteUint64(v uint64) { pos := len(w.buf) w.buf = append(w.buf, make([]byte, 8)...) binary.BigEndian.PutUint64(w.buf[pos:], v) } // WriteInt8 writes the given int8 to the buffer. func (w *BinaryWriter) WriteInt8(v int8) { w.WriteUint8(uint8(v)) } // WriteInt16 writes the given int16 to the buffer. func (w *BinaryWriter) WriteInt16(v int16) { w.WriteUint16(uint16(v)) } // WriteInt32 writes the given int32 to the buffer. func (w *BinaryWriter) WriteInt32(v int32) { w.WriteUint32(uint32(v)) } // WriteInt64 writes the given int64 to the buffer. func (w *BinaryWriter) WriteInt64(v int64) { w.WriteUint64(uint64(v)) } // BitmapReader is a binary bitmap reader. type BitmapReader struct { buf []byte pos uint32 // TODO: to uint64 eof bool } // NewBitmapReader returns a binary bitmap reader. func NewBitmapReader(buf []byte) *BitmapReader { return &BitmapReader{buf, 0, false} } // Pos returns the current bit position. func (r *BitmapReader) Pos() uint32 { return r.pos } // EOF returns if we reached the buffer's end-of-file. func (r *BitmapReader) EOF() bool { return r.eof } // Read reads the next bit. func (r *BitmapReader) Read() bool { if r.eof || uint32(len(r.buf)) <= (r.pos+1)/8 { r.eof = true return false } bit := r.buf[r.pos>>3]&(0x80>>(r.pos&7)) != 0 r.pos += 1 return bit } // BitmapWriter is a binary bitmap writer. type BitmapWriter struct { buf []byte pos uint32 } // NewBitmapWriter returns a binary bitmap writer. func NewBitmapWriter(buf []byte) *BitmapWriter { return &BitmapWriter{buf, 0} } // Len returns the buffer's length in bytes. func (w *BitmapWriter) Len() uint32 { return uint32(len(w.buf)) } // Bytes returns the buffer's bytes. func (w *BitmapWriter) Bytes() []byte { return w.buf } // Write writes the next bit. func (w *BitmapWriter) Write(bit bool) { if uint32(len(w.buf)) <= (w.pos+1)/8 { w.buf = append(w.buf, 0) } if bit { w.buf[w.pos>>3] = w.buf[w.pos>>3] | (0x80 >> (w.pos & 7)) } w.pos += 1 }