mirror of
https://github.com/superseriousbusiness/gotosocial.git
synced 2024-12-05 10:02:47 +00:00
413 lines
11 KiB
Go
413 lines
11 KiB
Go
|
// Copyright 2009 The Go Authors. All rights reserved.
|
||
|
// Use of this source code is governed by a BSD-style
|
||
|
// license that can be found in the LICENSE file.
|
||
|
|
||
|
package flate
|
||
|
|
||
|
import (
|
||
|
"math"
|
||
|
"math/bits"
|
||
|
)
|
||
|
|
||
|
const (
|
||
|
maxBitsLimit = 16
|
||
|
// number of valid literals
|
||
|
literalCount = 286
|
||
|
)
|
||
|
|
||
|
// hcode is a huffman code with a bit code and bit length.
|
||
|
type hcode uint32
|
||
|
|
||
|
func (h hcode) len() uint8 {
|
||
|
return uint8(h)
|
||
|
}
|
||
|
|
||
|
func (h hcode) code64() uint64 {
|
||
|
return uint64(h >> 8)
|
||
|
}
|
||
|
|
||
|
func (h hcode) zero() bool {
|
||
|
return h == 0
|
||
|
}
|
||
|
|
||
|
type huffmanEncoder struct {
|
||
|
codes []hcode
|
||
|
bitCount [17]int32
|
||
|
|
||
|
// Allocate a reusable buffer with the longest possible frequency table.
|
||
|
// Possible lengths are codegenCodeCount, offsetCodeCount and literalCount.
|
||
|
// The largest of these is literalCount, so we allocate for that case.
|
||
|
freqcache [literalCount + 1]literalNode
|
||
|
}
|
||
|
|
||
|
type literalNode struct {
|
||
|
literal uint16
|
||
|
freq uint16
|
||
|
}
|
||
|
|
||
|
// A levelInfo describes the state of the constructed tree for a given depth.
|
||
|
type levelInfo struct {
|
||
|
// Our level. for better printing
|
||
|
level int32
|
||
|
|
||
|
// The frequency of the last node at this level
|
||
|
lastFreq int32
|
||
|
|
||
|
// The frequency of the next character to add to this level
|
||
|
nextCharFreq int32
|
||
|
|
||
|
// The frequency of the next pair (from level below) to add to this level.
|
||
|
// Only valid if the "needed" value of the next lower level is 0.
|
||
|
nextPairFreq int32
|
||
|
|
||
|
// The number of chains remaining to generate for this level before moving
|
||
|
// up to the next level
|
||
|
needed int32
|
||
|
}
|
||
|
|
||
|
// set sets the code and length of an hcode.
|
||
|
func (h *hcode) set(code uint16, length uint8) {
|
||
|
*h = hcode(length) | (hcode(code) << 8)
|
||
|
}
|
||
|
|
||
|
func newhcode(code uint16, length uint8) hcode {
|
||
|
return hcode(length) | (hcode(code) << 8)
|
||
|
}
|
||
|
|
||
|
func reverseBits(number uint16, bitLength byte) uint16 {
|
||
|
return bits.Reverse16(number << ((16 - bitLength) & 15))
|
||
|
}
|
||
|
|
||
|
func maxNode() literalNode { return literalNode{math.MaxUint16, math.MaxUint16} }
|
||
|
|
||
|
func newHuffmanEncoder(size int) *huffmanEncoder {
|
||
|
// Make capacity to next power of two.
|
||
|
c := uint(bits.Len32(uint32(size - 1)))
|
||
|
return &huffmanEncoder{codes: make([]hcode, size, 1<<c)}
|
||
|
}
|
||
|
|
||
|
// Generates a HuffmanCode corresponding to the fixed literal table
|
||
|
func generateFixedLiteralEncoding() *huffmanEncoder {
|
||
|
h := newHuffmanEncoder(literalCount)
|
||
|
codes := h.codes
|
||
|
var ch uint16
|
||
|
for ch = 0; ch < literalCount; ch++ {
|
||
|
var bits uint16
|
||
|
var size uint8
|
||
|
switch {
|
||
|
case ch < 144:
|
||
|
// size 8, 000110000 .. 10111111
|
||
|
bits = ch + 48
|
||
|
size = 8
|
||
|
case ch < 256:
|
||
|
// size 9, 110010000 .. 111111111
|
||
|
bits = ch + 400 - 144
|
||
|
size = 9
|
||
|
case ch < 280:
|
||
|
// size 7, 0000000 .. 0010111
|
||
|
bits = ch - 256
|
||
|
size = 7
|
||
|
default:
|
||
|
// size 8, 11000000 .. 11000111
|
||
|
bits = ch + 192 - 280
|
||
|
size = 8
|
||
|
}
|
||
|
codes[ch] = newhcode(reverseBits(bits, size), size)
|
||
|
}
|
||
|
return h
|
||
|
}
|
||
|
|
||
|
func generateFixedOffsetEncoding() *huffmanEncoder {
|
||
|
h := newHuffmanEncoder(30)
|
||
|
codes := h.codes
|
||
|
for ch := range codes {
|
||
|
codes[ch] = newhcode(reverseBits(uint16(ch), 5), 5)
|
||
|
}
|
||
|
return h
|
||
|
}
|
||
|
|
||
|
var fixedLiteralEncoding = generateFixedLiteralEncoding()
|
||
|
var fixedOffsetEncoding = generateFixedOffsetEncoding()
|
||
|
|
||
|
func (h *huffmanEncoder) bitLength(freq []uint16) int {
|
||
|
var total int
|
||
|
for i, f := range freq {
|
||
|
if f != 0 {
|
||
|
total += int(f) * int(h.codes[i].len())
|
||
|
}
|
||
|
}
|
||
|
return total
|
||
|
}
|
||
|
|
||
|
func (h *huffmanEncoder) bitLengthRaw(b []byte) int {
|
||
|
var total int
|
||
|
for _, f := range b {
|
||
|
total += int(h.codes[f].len())
|
||
|
}
|
||
|
return total
|
||
|
}
|
||
|
|
||
|
// canReuseBits returns the number of bits or math.MaxInt32 if the encoder cannot be reused.
|
||
|
func (h *huffmanEncoder) canReuseBits(freq []uint16) int {
|
||
|
var total int
|
||
|
for i, f := range freq {
|
||
|
if f != 0 {
|
||
|
code := h.codes[i]
|
||
|
if code.zero() {
|
||
|
return math.MaxInt32
|
||
|
}
|
||
|
total += int(f) * int(code.len())
|
||
|
}
|
||
|
}
|
||
|
return total
|
||
|
}
|
||
|
|
||
|
// Return the number of literals assigned to each bit size in the Huffman encoding
|
||
|
//
|
||
|
// This method is only called when list.length >= 3
|
||
|
// The cases of 0, 1, and 2 literals are handled by special case code.
|
||
|
//
|
||
|
// list An array of the literals with non-zero frequencies
|
||
|
// and their associated frequencies. The array is in order of increasing
|
||
|
// frequency, and has as its last element a special element with frequency
|
||
|
// MaxInt32
|
||
|
// maxBits The maximum number of bits that should be used to encode any literal.
|
||
|
// Must be less than 16.
|
||
|
// return An integer array in which array[i] indicates the number of literals
|
||
|
// that should be encoded in i bits.
|
||
|
func (h *huffmanEncoder) bitCounts(list []literalNode, maxBits int32) []int32 {
|
||
|
if maxBits >= maxBitsLimit {
|
||
|
panic("flate: maxBits too large")
|
||
|
}
|
||
|
n := int32(len(list))
|
||
|
list = list[0 : n+1]
|
||
|
list[n] = maxNode()
|
||
|
|
||
|
// The tree can't have greater depth than n - 1, no matter what. This
|
||
|
// saves a little bit of work in some small cases
|
||
|
if maxBits > n-1 {
|
||
|
maxBits = n - 1
|
||
|
}
|
||
|
|
||
|
// Create information about each of the levels.
|
||
|
// A bogus "Level 0" whose sole purpose is so that
|
||
|
// level1.prev.needed==0. This makes level1.nextPairFreq
|
||
|
// be a legitimate value that never gets chosen.
|
||
|
var levels [maxBitsLimit]levelInfo
|
||
|
// leafCounts[i] counts the number of literals at the left
|
||
|
// of ancestors of the rightmost node at level i.
|
||
|
// leafCounts[i][j] is the number of literals at the left
|
||
|
// of the level j ancestor.
|
||
|
var leafCounts [maxBitsLimit][maxBitsLimit]int32
|
||
|
|
||
|
// Descending to only have 1 bounds check.
|
||
|
l2f := int32(list[2].freq)
|
||
|
l1f := int32(list[1].freq)
|
||
|
l0f := int32(list[0].freq) + int32(list[1].freq)
|
||
|
|
||
|
for level := int32(1); level <= maxBits; level++ {
|
||
|
// For every level, the first two items are the first two characters.
|
||
|
// We initialize the levels as if we had already figured this out.
|
||
|
levels[level] = levelInfo{
|
||
|
level: level,
|
||
|
lastFreq: l1f,
|
||
|
nextCharFreq: l2f,
|
||
|
nextPairFreq: l0f,
|
||
|
}
|
||
|
leafCounts[level][level] = 2
|
||
|
if level == 1 {
|
||
|
levels[level].nextPairFreq = math.MaxInt32
|
||
|
}
|
||
|
}
|
||
|
|
||
|
// We need a total of 2*n - 2 items at top level and have already generated 2.
|
||
|
levels[maxBits].needed = 2*n - 4
|
||
|
|
||
|
level := uint32(maxBits)
|
||
|
for level < 16 {
|
||
|
l := &levels[level]
|
||
|
if l.nextPairFreq == math.MaxInt32 && l.nextCharFreq == math.MaxInt32 {
|
||
|
// We've run out of both leafs and pairs.
|
||
|
// End all calculations for this level.
|
||
|
// To make sure we never come back to this level or any lower level,
|
||
|
// set nextPairFreq impossibly large.
|
||
|
l.needed = 0
|
||
|
levels[level+1].nextPairFreq = math.MaxInt32
|
||
|
level++
|
||
|
continue
|
||
|
}
|
||
|
|
||
|
prevFreq := l.lastFreq
|
||
|
if l.nextCharFreq < l.nextPairFreq {
|
||
|
// The next item on this row is a leaf node.
|
||
|
n := leafCounts[level][level] + 1
|
||
|
l.lastFreq = l.nextCharFreq
|
||
|
// Lower leafCounts are the same of the previous node.
|
||
|
leafCounts[level][level] = n
|
||
|
e := list[n]
|
||
|
if e.literal < math.MaxUint16 {
|
||
|
l.nextCharFreq = int32(e.freq)
|
||
|
} else {
|
||
|
l.nextCharFreq = math.MaxInt32
|
||
|
}
|
||
|
} else {
|
||
|
// The next item on this row is a pair from the previous row.
|
||
|
// nextPairFreq isn't valid until we generate two
|
||
|
// more values in the level below
|
||
|
l.lastFreq = l.nextPairFreq
|
||
|
// Take leaf counts from the lower level, except counts[level] remains the same.
|
||
|
if true {
|
||
|
save := leafCounts[level][level]
|
||
|
leafCounts[level] = leafCounts[level-1]
|
||
|
leafCounts[level][level] = save
|
||
|
} else {
|
||
|
copy(leafCounts[level][:level], leafCounts[level-1][:level])
|
||
|
}
|
||
|
levels[l.level-1].needed = 2
|
||
|
}
|
||
|
|
||
|
if l.needed--; l.needed == 0 {
|
||
|
// We've done everything we need to do for this level.
|
||
|
// Continue calculating one level up. Fill in nextPairFreq
|
||
|
// of that level with the sum of the two nodes we've just calculated on
|
||
|
// this level.
|
||
|
if l.level == maxBits {
|
||
|
// All done!
|
||
|
break
|
||
|
}
|
||
|
levels[l.level+1].nextPairFreq = prevFreq + l.lastFreq
|
||
|
level++
|
||
|
} else {
|
||
|
// If we stole from below, move down temporarily to replenish it.
|
||
|
for levels[level-1].needed > 0 {
|
||
|
level--
|
||
|
}
|
||
|
}
|
||
|
}
|
||
|
|
||
|
// Somethings is wrong if at the end, the top level is null or hasn't used
|
||
|
// all of the leaves.
|
||
|
if leafCounts[maxBits][maxBits] != n {
|
||
|
panic("leafCounts[maxBits][maxBits] != n")
|
||
|
}
|
||
|
|
||
|
bitCount := h.bitCount[:maxBits+1]
|
||
|
bits := 1
|
||
|
counts := &leafCounts[maxBits]
|
||
|
for level := maxBits; level > 0; level-- {
|
||
|
// chain.leafCount gives the number of literals requiring at least "bits"
|
||
|
// bits to encode.
|
||
|
bitCount[bits] = counts[level] - counts[level-1]
|
||
|
bits++
|
||
|
}
|
||
|
return bitCount
|
||
|
}
|
||
|
|
||
|
// Look at the leaves and assign them a bit count and an encoding as specified
|
||
|
// in RFC 1951 3.2.2
|
||
|
func (h *huffmanEncoder) assignEncodingAndSize(bitCount []int32, list []literalNode) {
|
||
|
code := uint16(0)
|
||
|
for n, bits := range bitCount {
|
||
|
code <<= 1
|
||
|
if n == 0 || bits == 0 {
|
||
|
continue
|
||
|
}
|
||
|
// The literals list[len(list)-bits] .. list[len(list)-bits]
|
||
|
// are encoded using "bits" bits, and get the values
|
||
|
// code, code + 1, .... The code values are
|
||
|
// assigned in literal order (not frequency order).
|
||
|
chunk := list[len(list)-int(bits):]
|
||
|
|
||
|
sortByLiteral(chunk)
|
||
|
for _, node := range chunk {
|
||
|
h.codes[node.literal] = newhcode(reverseBits(code, uint8(n)), uint8(n))
|
||
|
code++
|
||
|
}
|
||
|
list = list[0 : len(list)-int(bits)]
|
||
|
}
|
||
|
}
|
||
|
|
||
|
// Update this Huffman Code object to be the minimum code for the specified frequency count.
|
||
|
//
|
||
|
// freq An array of frequencies, in which frequency[i] gives the frequency of literal i.
|
||
|
// maxBits The maximum number of bits to use for any literal.
|
||
|
func (h *huffmanEncoder) generate(freq []uint16, maxBits int32) {
|
||
|
list := h.freqcache[:len(freq)+1]
|
||
|
codes := h.codes[:len(freq)]
|
||
|
// Number of non-zero literals
|
||
|
count := 0
|
||
|
// Set list to be the set of all non-zero literals and their frequencies
|
||
|
for i, f := range freq {
|
||
|
if f != 0 {
|
||
|
list[count] = literalNode{uint16(i), f}
|
||
|
count++
|
||
|
} else {
|
||
|
codes[i] = 0
|
||
|
}
|
||
|
}
|
||
|
list[count] = literalNode{}
|
||
|
|
||
|
list = list[:count]
|
||
|
if count <= 2 {
|
||
|
// Handle the small cases here, because they are awkward for the general case code. With
|
||
|
// two or fewer literals, everything has bit length 1.
|
||
|
for i, node := range list {
|
||
|
// "list" is in order of increasing literal value.
|
||
|
h.codes[node.literal].set(uint16(i), 1)
|
||
|
}
|
||
|
return
|
||
|
}
|
||
|
sortByFreq(list)
|
||
|
|
||
|
// Get the number of literals for each bit count
|
||
|
bitCount := h.bitCounts(list, maxBits)
|
||
|
// And do the assignment
|
||
|
h.assignEncodingAndSize(bitCount, list)
|
||
|
}
|
||
|
|
||
|
// atLeastOne clamps the result between 1 and 15.
|
||
|
func atLeastOne(v float32) float32 {
|
||
|
if v < 1 {
|
||
|
return 1
|
||
|
}
|
||
|
if v > 15 {
|
||
|
return 15
|
||
|
}
|
||
|
return v
|
||
|
}
|
||
|
|
||
|
func histogram(b []byte, h []uint16) {
|
||
|
if true && len(b) >= 8<<10 {
|
||
|
// Split for bigger inputs
|
||
|
histogramSplit(b, h)
|
||
|
} else {
|
||
|
h = h[:256]
|
||
|
for _, t := range b {
|
||
|
h[t]++
|
||
|
}
|
||
|
}
|
||
|
}
|
||
|
|
||
|
func histogramSplit(b []byte, h []uint16) {
|
||
|
// Tested, and slightly faster than 2-way.
|
||
|
// Writing to separate arrays and combining is also slightly slower.
|
||
|
h = h[:256]
|
||
|
for len(b)&3 != 0 {
|
||
|
h[b[0]]++
|
||
|
b = b[1:]
|
||
|
}
|
||
|
n := len(b) / 4
|
||
|
x, y, z, w := b[:n], b[n:], b[n+n:], b[n+n+n:]
|
||
|
y, z, w = y[:len(x)], z[:len(x)], w[:len(x)]
|
||
|
for i, t := range x {
|
||
|
v0 := &h[t]
|
||
|
v1 := &h[y[i]]
|
||
|
v3 := &h[w[i]]
|
||
|
v2 := &h[z[i]]
|
||
|
*v0++
|
||
|
*v1++
|
||
|
*v2++
|
||
|
*v3++
|
||
|
}
|
||
|
}
|