godzil/quickdev16
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add huffman test

Browse Source
This commit is contained in:
optixx 2009-08-28 08:48:21 +02:00
parent b1db3f6ae0
commit 44df97f81a
3 changed files with 704 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
tools/huffman/Makefile Normal file
View File

@ -0,0 +1,10 @@
SHELL = sh
OPTIMIZE = -O2
CC = gcc
CSTD = c99
override CFLAGS = -pedantic -std=$(CSTD) -Wall -Wstrict-prototypes $(OPTIMIZE)
all: huffman-decode huffman-encode
clean:
rm -f huffman-decode huffman-encode bin2avrS

224
tools/huffman/huffman-decode.c Normal file
View File

@ -0,0 +1,224 @@
/* huffman-decode.c */
/*
This file is part of the AVR-Huffman.
Copyright (C) 2009 Daniel Otte (daniel.otte@rub.de)
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <stdio.h>
#include <string.h>
#include <stdint.h>
#include <stdlib.h>
#define V_NODE (-2)
#define V_EOF (-1)
#define PREFIX_SIZE_B 32
#define ALLOC_ERROR {fprintf(stderr,"failed to alloc memory in %s @ %d !\n",__FILE__, __LINE__); exit(-1);}
#define DEBUG 0
typedef struct {
int16_t value;
void* left;
void* right;
} node_t;
node_t* tree=NULL;
uint16_t tree_index;
void prefix_increment(uint8_t* prefix){
uint8_t i;
for(i=0; i<PREFIX_SIZE_B; ++i){
prefix[i] += 1;
if(prefix[i]!=0)
return;
}
}
void prefix_shiftleft(uint8_t* prefix){
uint8_t i;
uint8_t c[2]={0,0};
uint8_t ci=0;
for(i=0; i<PREFIX_SIZE_B; ++i){
c[ci] = (prefix[i])>>7;
prefix[i]<<=1;
ci ^= 1;
prefix[i]|=c[ci];
}
}
uint8_t append_tree(int16_t value, uint16_t depth){
static uint8_t prefix[PREFIX_SIZE_B];
static uint8_t cdepth=0;
node_t* current=tree;
int8_t i,t;
for(;cdepth<depth;++cdepth){
prefix_shiftleft(prefix);
}
for(i=depth-1; i>=0; --i){
t=(prefix[i/8])&(1<<(i%8));
if(t==0){
if(current->left==NULL){
// current->left=calloc(1, sizeof(node_t));
current->left=&(tree[tree_index++]);
((node_t*)(current->left))->value = V_NODE;
}
current = current->left;
} else {
if(current->right==NULL){
// current->right=calloc(1, sizeof(node_t));
current->right=&(tree[tree_index++]);
((node_t*)(current->right))->value = V_NODE;
}
current = current->right;
}
}
if(current==NULL)
ALLOC_ERROR
current->value=value;
prefix_increment(prefix);
return prefix[depth/8]&(1<<(depth%8));
}
void set_last_to_eof(void){
node_t* current=tree;
while(current->value==V_NODE)
current=current->right;
current->value=V_EOF;
}
void build_tree(FILE* f){
uint16_t treesize;
uint16_t x1, x2;
x1 =fgetc(f);
x2 =fgetc(f);
if(x1!=0xc0 || (x2&0xFE)!=0xde){
fprintf(stderr,"No magic values found!\n");
}
treesize= fgetc(f);
if(treesize>0xff){
fprintf(stderr,"You are trying to uncompress an empty file!\n");
exit(-3);
}
treesize = 2*(((x2&1)<<8)+treesize)-1;
//printf("Using treesize=%i with size of %i\n", treesize, sizeof(node_t));
tree = calloc(treesize, sizeof(node_t));
if(tree==NULL)
ALLOC_ERROR
tree->value = V_NODE;
tree_index=1;
uint16_t depth=0;
uint16_t count=0;
uint8_t v;
do{
while(count==0){
depth++;
count= fgetc(f);
if(count==255)
count += fgetc(f);
}
v = fgetc(f);
--count;
}while(!append_tree(v, depth));
set_last_to_eof();
}
void free_tree(node_t* node){
free(node);
}
FILE* outfile;
FILE* infile;
void write_char(uint8_t v){
fputc(v, outfile);
}
uint8_t read_bit(void){
static uint8_t buffer;
static uint8_t index=0;
uint8_t t;
if(index==0){
buffer = fgetc(infile);
if(feof(infile))
return 0xFF;
}
t=buffer>>7;
buffer<<=1;
index = (index+1)%8;
return t;
}
uint16_t decompress_byte(void){
node_t* current=tree;
uint8_t t;
while(current->value==V_NODE){
t=read_bit();
if(t==0xFF)
return EOF;
if(t==0){
current=current->left;
} else {
current=current->right;
}
if(current==NULL){
fprintf(stderr, "Tree damaged!\n");
exit(-4);
}
}
return current->value;
}
void decompress(void){
uint16_t t;
for(;;){
t=decompress_byte();
if(t==(uint16_t)EOF)
return;
write_char(t);
}
}
int main(int argc, char** argv){
int i;
FILE* fin;
FILE* fg;
// outfile=stdout;
for(i=1;i<argc;++i){
fin = fopen(argv[i], "r");
fg = fopen("graphout.dot", "w");
outfile = fopen("decompress.out", "w");
puts("building tree ...");
build_tree(fin);
fclose(fg);
infile=fin;
puts("decompression ...");
decompress();
fclose(fin);
fclose(outfile);
free_tree(tree);
}
return 0;
}

470
tools/huffman/huffman-encode.c Normal file
View File

@ -0,0 +1,470 @@
/* huffman-encode.c */
/*
This file is part of the AVR-Huffman.
Copyright (C) 2009 Daniel Otte (daniel.otte@rub.de)
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <stdio.h>
#include <string.h>
#include <stdint.h>
#include <stdlib.h>
#define DEBUG 0
#define XDEBUG 0
typedef struct{
uint8_t depth;
uint16_t value;
uint32_t weight;
void* left;
void* right;
} node_t;
typedef struct{
uint16_t value;
unsigned depth;
void* encoding;
} item_t;
typedef struct{
uint8_t initialized;
item_t* item;
void* parent;
void* left;
void* right;
} node2_t;
uint32_t histogram[256];
node_t* pool[256+1];
unsigned poolsize;
unsigned item_count;
node_t* tree;
node_t* treenodes;
unsigned treeindex=0;
item_t* itemlist;
unsigned itemindex=0;
node2_t* node2list=NULL;
unsigned node2list_index=0;
item_t* valueencode[256];
item_t* eof_encoding;
void reset_histogram(void){
memset(histogram, 0, 256*sizeof(uint32_t));
}
void build_histogram(char* fname){
FILE* f;
int t;
f = fopen(fname, "r");
while((t=fgetc(f))!=EOF){
histogram[(uint8_t)t&0xFF]++;
}
fclose(f);
}
void build_pool(void){
unsigned i,j;
memset(pool, 0, 256*sizeof(node_t*));
for(i=0,j=0;i<256;++i){
if(histogram[i]==0)
continue;
pool[j] = malloc(sizeof(node_t));
if(pool[j]==NULL){
fprintf(stderr,"out of memory error (%d)!\n", __LINE__);
exit(-1);
}
pool[j]->depth = 0;
pool[j]->value = i;
pool[j]->left = NULL;
pool[j]->right = NULL;
pool[j]->weight = histogram[i];
j++;
}
pool[j] = malloc(sizeof(node_t));
if(pool[j]==NULL){
fprintf(stderr,"out of memory error (%d)!\n", __LINE__);
exit(-1);
}
pool[j]->depth = 0;
pool[j]->value = 0xFFFF;
pool[j]->left = NULL;
pool[j]->right = NULL;
pool[j]->weight = 1;
j++;
poolsize = j;
item_count = j;
}
void find_lightest2(unsigned* a, unsigned* b, unsigned* depth){
unsigned ia=0;
unsigned ib=0;
uint32_t wa, wb;
unsigned i;
wa = wb = 0xFFFFFFFF;
for(i=0; i<poolsize; ++i){
if(pool[i]==NULL)
continue;
if(wa>=pool[i]->weight){
wb = wa;
ib = ia;
wa = pool[i]->weight;
ia = i;
}else{
if(wb>pool[i]->weight){
wb = pool[i]->weight;
ib = i;
}
}
}
if(wb == 0xFFFFFFFF || wa == 0xFFFFFFFF){
fprintf(stderr, "Error while searching!\n");
exit(-2);
}
if(pool[ia]->depth <= pool[ib]->depth){
*a = ia;
*b = ib;
*depth = pool[ib]->depth;
} else {
*a = ib;
*b = ia;
*depth = pool[ia]->depth;
}
}
void init_tree(void){
treenodes = malloc((poolsize-1)*sizeof(node_t));
if(treenodes==NULL){
fprintf(stderr,"out of memory error (%d)!\n", __LINE__);
exit(-1);
}
#if XDEBUG
printf("treenodes := %p\n", (void*)treenodes);
#endif
}
void update_tree(void){
if(poolsize<2)
return;
unsigned a,b, depth;
find_lightest2(&a,&b,&depth);
#if XDEBUG
printf("joining %d and %d\n", a,b);
#endif
treenodes[treeindex].depth = depth+1;
treenodes[treeindex].weight = pool[a]->weight + pool[b]->weight;
treenodes[treeindex].left = (pool[a]);
treenodes[treeindex].right = (pool[b]);
#if XDEBUG
printf(" idx = %d\n self = %p\n depth = %d\n weight = %d\n"
" left = %p\n right = %p\n",
treeindex, (void*)&(treenodes[treeindex]), treenodes[treeindex].depth,
treenodes[treeindex].weight, treenodes[treeindex].left,
treenodes[treeindex].right);
#endif
pool[a] = &(treenodes[treeindex]);
pool[b] = pool[poolsize-1];
pool[poolsize-1] = NULL;
--poolsize;
++treeindex;
}
void build_tree(void){
while(poolsize>1){
update_tree();
}
tree = &(treenodes[treeindex-1]);
}
void free_leaf(node_t* node){
if(node->depth==0){
free(node);
}else{
free_leaf(node->left);
free_leaf(node->right);
}
}
void free_tree(void){
free_leaf(tree);
free(treenodes);
tree = NULL;
}
void init_itemlist(void){
itemlist = calloc((item_count),sizeof(item_t));
if(itemlist==NULL){
fprintf(stderr,"out of memory error (%d)!\n", __LINE__);
exit(-1);
}
}
void update_itemlist(node_t* node, unsigned depth){
if(node->depth==0){
itemlist[itemindex].value = node->value;
itemlist[itemindex].depth = depth;
++itemindex;
} else {
update_itemlist(node->left, depth+1);
update_itemlist(node->right, depth+1);
}
}
void build_itemlist(void){
update_itemlist(tree, 0);
}
int item_compare_depth(const void* a, const void* b){
if(((item_t*)a)->value==0xFFFF)
return 1;
if(((item_t*)b)->value==0xFFFF)
return -1;
return ((item_t*)a)->depth - ((item_t*)b)->depth;
}
void sort_itemlist(void){
qsort(itemlist, item_count, sizeof(item_t), item_compare_depth);
}
#define PREFIX_SIZE_B 32
void prefix_increment(uint8_t* prefix){
uint8_t i;
for(i=0; i<PREFIX_SIZE_B; ++i){
prefix[i] += 1;
if(prefix[i]!=0)
return;
}
}
void prefix_shiftleft(uint8_t* prefix){
uint8_t i;
uint8_t c[2]={0,0};
uint8_t ci=0;
for(i=0; i<PREFIX_SIZE_B; ++i){
c[ci] = (prefix[i])>>7;
prefix[i]<<=1;
ci ^= 1;
prefix[i]|=c[ci];
}
}
void gen_itemencoding(void){
uint8_t prefix[PREFIX_SIZE_B];
memset(prefix, 0, PREFIX_SIZE_B);
unsigned depth=0;
unsigned depth_B=0;
unsigned i,j;
for(i=0; i<item_count; ++i){
if(depth!=itemlist[i].depth){
for(j=depth; j<itemlist[i].depth; ++j)
prefix_shiftleft(prefix);
depth = itemlist[i].depth;
depth_B = (depth+7)/8;
}
itemlist[i].encoding=malloc(depth_B);
if(itemlist[i].encoding==NULL){
fprintf(stderr,"out of memory error (%d)!\n", __LINE__);
exit(-1);
}
memcpy(itemlist[i].encoding, prefix, depth_B);
prefix_increment(prefix);
}
}
void add_item2(item_t* item){
int i;
uint8_t t;
node2_t* current;
current = node2list;
char c;
c = item->value;
// printf(" %2.2X (%c) => ", c, (c>32&&c<128)?c:' ');
for(i=item->depth-1; i>=0; --i){
t = (((uint8_t*)(item->encoding))[i/8])&(1<<(i%8));
if(current->initialized==0){
current->initialized=1;
current->parent = NULL;
current->item = NULL;
current->left = NULL;
current->right = NULL;
node2list_index++;
}
if(t==0){
// putchar('0');
if(current->left){
current = current->left;
} else {
current->left = &(node2list[node2list_index++]);
((node2_t*)(current->left))->parent = current;
current = current->left;
current->initialized=1;
current->item=NULL;
}
} else {
// putchar('1');
if(current->right){
current = current->right;
} else {
current->right = &(node2list[node2list_index++]);
((node2_t*)(current->right))->parent = current;
current = current->right;
current->initialized=1;
current->item=NULL;
}
}
}
current->item = item;
current->left = NULL;
current->right = NULL;
// printf("\n");
}
void gen_tree2(void){
unsigned i;
node2list = calloc(2*item_count-1, sizeof(node2_t));
#if XDEBUG
printf("item_count = %d\n", item_count);
#endif
if(node2list==NULL){
fprintf(stderr,"out of memory error (%d)!\n", __LINE__);
exit(-1);
}
for(i=0;i<item_count;++i){
add_item2(&(itemlist[i]));
}
}
void bit_writer(FILE* f, uint8_t bit, uint8_t flush){
static uint8_t buffer;
static uint8_t buffer_index=0;
if(flush){
for(;buffer_index<8;++buffer_index)
buffer<<=1;
}
if(buffer_index==8){
fputc(buffer, f);
buffer_index=0;
}
buffer<<=1;
buffer|=bit?1:0;
buffer_index++;
}
void encoding_writer(FILE* f, uint8_t* data, uint16_t length){
int i;
if(data==NULL)
bit_writer(f, 0, 1);
for(i=length-1;i>=0;--i){
bit_writer(f,data[i/8]&(1<<(i%8)),0);
}
}
void build_valueencode(void){
unsigned i;
memset(valueencode, 0, 256*sizeof(void*));
for(i=0; i<item_count-1; ++i){
valueencode[itemlist[i].value] = &(itemlist[i]);
}
eof_encoding = &(itemlist[i]);
}
void write_tree(FILE* f){
unsigned i,j;
unsigned last=0;
unsigned last_depth=1;
fputc(0xc0, f);
fputc(0xde + (item_count>>8), f);
fputc(item_count,f);
for(i=0; i<item_count; ++i){
if(itemlist[i].depth!=last_depth){
if(i-last>=255){
fputc(255, f);
fputc(i-last-255, f);
} else {
fputc(i-last, f);
}
for(j=last; j<i; ++j){
fputc(itemlist[j].value, f);
}
for(j=last_depth+1; j<itemlist[i].depth; ++j)
fputc(0, f);
last=i;
last_depth=itemlist[i].depth;
}
}
if(i-last>=255){
fputc(255, f);
fputc(i-last-255, f);
} else {
fputc(i-last, f);
}
for(j=last; j<i; ++j){
fputc(itemlist[j].value, f);
}
}
void compress_file(FILE* fin, FILE* fout){
int t;
while((t=fgetc(fin))!=EOF){
t = (uint8_t)t;
if(valueencode[t]==NULL){
fprintf(stderr,"no encoding for %2.2X (%c) found!\n",t,(t>32&&t<128)?t:' ');
exit(-3);
}
encoding_writer(fout, valueencode[t]->encoding, valueencode[t]->depth);
};
encoding_writer(fout, eof_encoding->encoding, eof_encoding->depth);
encoding_writer(fout, NULL, 0);
}
void compress_huffman(char * filename_in, char * filename_out){
FILE* fin;
FILE* fout;
reset_histogram();
build_histogram(filename_in);
build_pool();
init_tree();
build_tree();
init_itemlist();
build_itemlist();
free_tree();
sort_itemlist();
gen_itemencoding();
gen_tree2();
build_valueencode();
fin = fopen(filename_in, "r");
fout = fopen(filename_out, "w");
write_tree(fout);
compress_file(fin, fout);
fclose(fin);
fclose(fout);
}
int main(int argc, char** argv){
char filename_out[strlen(argv[1])+20];
strcpy(filename_out, argv[1]);
strcat(filename_out, ".hfm");
compress_huffman(argv[1],filename_out);
return 0;
}