charconv.tcl

Go to the documentation of this file.

#!/usr/bin/tclsh
 
proc usage {} {
    puts {charconv.tcl: [-p prefix] [-s split] [-o ofile] file ... }
    exit 1
}
 
proc preamble_trie {ofilehandle ifiles ofile} {
    set f $ofilehandle
 
    set totype {unsigned }
 
    puts $f "/** \\file $ofile"
    puts $f "    \\brief Character conversion, generated from [lindex $ifiles 0]"
    puts $f ""
    puts $f "    Generated automatically by charconv.tcl"
    puts $f "*/"
    puts $f "\#include <string.h>"
 
    puts $f "\#if HAVE_CONFIG_H"
    puts $f "\#include <config.h>"
    puts $f "\#endif"
 
    puts $f "
        struct yaz_iconv_trie_flat {
            char from\[6\];
            unsigned combining : 1;
            $totype to : 24;
        };
        struct yaz_iconv_trie_dir {
            int ptr : 15;
            unsigned combining : 1;
            $totype to : 24;
        };
 
        struct yaz_iconv_trie {
            struct yaz_iconv_trie_flat *flat;
            struct yaz_iconv_trie_dir *dir;
        };
    "
    puts $f {
        static unsigned long lookup(struct yaz_iconv_trie **ptrs, int ptr, unsigned char *inp,
                                    size_t inbytesleft, size_t *no_read, int *combining, unsigned mask, unsigned int boffset)
        {
            struct yaz_iconv_trie *t = ptrs[ptr-1];
            if (inbytesleft < 1)
                return 0;
            if (t->dir)
            {
                size_t ch = (inp[0] & mask) + boffset;
                unsigned long code;
                if (t->dir[ch].ptr)
                {
                    code = lookup(ptrs, t->dir[ch].ptr, inp+1, inbytesleft-1, no_read, combining, mask, boffset);
                    if (code)
                    {
                        (*no_read)++;
                        return code;
                    }
                }
                if (t->dir[ch].to)
                {
                    code = t->dir[ch].to;
                    *combining = t->dir[ch].combining;
                    *no_read = 1;
                    return code;
                }
            }
            else
            {
                struct yaz_iconv_trie_flat *flat = t->flat;
                while (flat->to)
                {
                    size_t len = strlen(flat->from);
                    if (len <= inbytesleft)
                    {
                        size_t i;
                        for (i = 0; i < len; i++)
                        {
                            if (((unsigned char *) flat->from)[i] != (inp[i] & mask) + boffset)
                                break;
                        }
                        if (i == len)
                        {
                            *no_read = len;
                            *combining = flat->combining;
                            return flat->to;
                        }
                    }
                    flat++;
                }
            }
            return 0;
        }
    }
}
 
proc reset_trie {} {
    global trie
 
    foreach x [array names trie] {
        unset trie($x)
    }
 
    set trie(no) 1
    set trie(size) 0
    set trie(max) 0
    set trie(split) 50
    set trie(prefix) {}
}
 
proc ins_trie {from to combining codename} {
    global trie
    if {![info exists trie(no)]} {
        set trie(no) 1
        set trie(size) 0
        set trie(max) 0
    }
    if {$trie(max) < $to} {
        set trie(max) $to
    }
    incr trie(size)
    ins_trie_r [split $from] $to $combining $codename 0
}
 
proc split_trie {this} {
    global trie
    set trie($this,type) d
    foreach e $trie($this,content) {
        set from [lindex $e 0]
        set to [lindex $e 1]
        set combining [lindex $e 2]
        set codename [lindex $e 3]
 
        set ch [lindex $from 0]
        set rest [lrange $from 1 end]
 
        if {[llength $rest]} {
            if {![info exist trie($this,ptr,$ch)]} {
                set trie($this,ptr,$ch) $trie(no)
                incr trie(no)
            }
            ins_trie_r $rest $to $combining $codename $trie($this,ptr,$ch)
        } else {
            set trie($this,to,$ch) $to
            set trie($this,combining,$ch) $combining
            set trie($this,codename,$ch) $codename
        }
    }
    set trie($this,content) missing
}
 
proc ins_trie_r {from to combining codename this} {
    global trie
 
    if {![info exist trie($this,type)]} {
        set trie($this,type) f
    }
    if {$trie($this,type) == "f"} {
        set dup 0
        if {[info exists trie($this,content)]} {
            foreach e $trie($this,content) {
                set efrom [lindex $e 0]
                if { $efrom == $from } {
                    set dup 1
                }
            }
        }
        if { $dup == 0 } {
            lappend trie($this,content) [list $from $to $combining $codename]
        }
 
        # split ?
        if {[llength $trie($this,content)] > $trie(split)} {
            split_trie $this
            return [ins_trie_r $from $to $combining $codename $this]
        }
    } else {
        set ch [lindex $from 0]
        set rest [lrange $from 1 end]
 
        if {[llength $rest]} {
            if {![info exist trie($this,ptr,$ch)]} {
                set trie($this,ptr,$ch) $trie(no)
                incr trie(no)
            }
            ins_trie_r $rest $to $combining $codename $trie($this,ptr,$ch)
        } else {
            if {![info exist trie($this,to,$ch)]} {
                set trie($this,to,$ch) $to
                set trie($this,combining,$ch) $combining
                set trie($this,codename,$ch) $codename
            }
        }
    }
}
 
proc dump_trie {ofilehandle} {
    global trie
 
    set f $ofilehandle
 
    puts $f "/* TRIE: size $trie(size) */"
 
    set this $trie(no)
    while { [incr this -1] >= 0 } {
        puts $f "/* PAGE $this */"
        if {$trie($this,type) == "f"} {
            puts $f "struct yaz_iconv_trie_flat $trie(prefix)page${this}_flat\[\] = \{"
            foreach m $trie($this,content) {
                puts -nonewline $f "  \{\""
                foreach d [lindex $m 0] {
                    puts -nonewline $f "\\x$d"
                }
                puts -nonewline $f "\", [lindex $m 2], 0x[lindex $m 1]"
                set v [lindex $m 3]
                puts $f "\}, /* $v */"
            }
            puts $f "  \{\"\", 0, 0\}"
            puts $f "\};"
            puts $f "struct yaz_iconv_trie $trie(prefix)page${this} = \{"
            puts $f "  $trie(prefix)page${this}_flat, 0"
            puts $f "\};"
        } else {
            puts $f "struct yaz_iconv_trie_dir $trie(prefix)page${this}_dir\[256\] = \{"
            for {set i 0} {$i < 256} {incr i} {
                puts -nonewline $f "  \{"
                set ch [format %02X $i]
                set null 1
                if {[info exist trie($this,ptr,$ch)]} {
                    puts -nonewline $f "[expr $trie($this,ptr,$ch)+1], "
                    set null 0
                } else {
                    puts -nonewline $f "0, "
                }
                if {[info exist trie($this,combining,$ch)]} {
                    puts -nonewline $f "$trie($this,combining,$ch), "
                } else {
                    puts -nonewline $f "0, "
                }
                if {[info exist trie($this,to,$ch)]} {
                    puts -nonewline $f "0x$trie($this,to,$ch)\}"
                    set null 0
                } else {
                    puts -nonewline $f "0\}"
                }
                if {[info exist trie($this,codename,$ch)]} {
                    set v $trie($this,codename,$ch)
                    puts -nonewline $f " /* $v */"
                }
                if {$i < 255} {
                    puts $f ","
                } else {
                    puts $f ""
                }
            }
            puts $f "\};"
            puts $f "struct yaz_iconv_trie $trie(prefix)page${this} = \{"
            puts $f "  0, $trie(prefix)page${this}_dir"
            puts $f "\};"
        }
    }
 
    puts $f "struct yaz_iconv_trie *$trie(prefix)ptrs \[\] = {"
    for {set this 0} {$this < $trie(no)} {incr this} {
        puts $f " &$trie(prefix)page$this,"
    }
    puts $f "0, };"
    puts $f ""
 
    puts $f "unsigned long yaz_$trie(prefix)_conv
            (unsigned char *inp, size_t inbytesleft, size_t *no_read, int *combining, unsigned mask, unsigned int boffset)
        {
            unsigned long code;
 
            code = lookup($trie(prefix)ptrs, 1, inp, inbytesleft, no_read, combining, mask, boffset);
            if (!code)
            {
                *no_read = 1;
            }
            return code;
        }
    "
}
 
proc readfile {fname ofilehandle prefix omits reverse} {
    global trie
 
    set marc_lines 0
    set ucs_lines 0
    set utf_lines 0
    set altutf_lines 0
    set codename_lines 0
    set lineno 0
    set f [open $fname r]
    set tablenumber x
    set combining 0
    set codename {}
    set altutf {}
    while {1} {
        incr lineno
        set cnt [gets $f line]
        if {$cnt < 0} {
            break
        }
        if {[regexp {</characterSet>} $line s]} {
            dump_trie $ofilehandle
        } elseif {[regexp {<characterSet .*ISOcode="([0-9A-Fa-f]+)"} $line s tablenumber]} {
            reset_trie
            set trie(prefix) "${prefix}_$tablenumber"
            set combining 0
        } elseif {[regexp {</code>} $line s]} {
            if {[string length $ucs]} {
                if {$reverse} {
                    for {set i 0} {$i < [string length $utf]} {incr i 2} {
                        lappend hex [string range $utf $i [expr $i+1]]
                    }
                    # puts "ins_trie $hex $marc
                    ins_trie $hex $marc $combining $codename
                    unset hex
 
                } else {
                    for {set i 0} {$i < [string length $marc]} {incr i 2} {
                        lappend hex [string range $marc $i [expr $i+1]]
                    }
                    # puts "ins_trie $hex $ucs"
                    ins_trie $hex $ucs $combining $codename
                    unset hex
                }
            }
            if {$reverse && [string length $marc]} {
                for {set i 0} {$i < [string length $altutf]} {incr i 2} {
                    lappend hex [string range $altutf $i [expr $i+1]]
                }
                if {[info exists hex]} {
                    ins_trie $hex $marc $combining $codename
                    unset hex
                }
            }
            set marc {}
            set uni {}
            set codename {}
            set combining 0
            set altutf {}
        } elseif {[regexp {<marc>([0-9A-Fa-f]*)</marc>} $line s marc]} {
            incr marc_lines
        } elseif {[regexp {<name>(.*)</name>} $line s codename]} {
            incr codename_lines
        } elseif {[regexp {<name>(.*)} $line s codename]} {
            incr codename_lines
            incr lineno
            set cnt [gets $f line]
            if {$cnt < 0} {
                break
            }
            if {[regexp {(.*)</name>} $line s codename_ex]} {
                set codename "${codename} ${codename_ex}"
            }
        } elseif {[regexp {<isCombining>true</isCombining>} $line s]} {
            set combining 1
        } elseif {[regexp {<ucs>([0-9A-Fa-f]*)</ucs>} $line s ucs]} {
            incr ucs_lines
        } elseif {[regexp {<utf-8>([0-9A-Fa-f]*)</utf-8>} $line s utf]} {
            incr utf_lines
        } elseif {[regexp {<altutf-8>([0-9A-Fa-f]*)</altutf-8>} $line s altutf]} {
            incr altutf_lines
        }
    }
    close $f
}
 
set verbose 0
set ifile {}
set ofile out.c
set prefix {c}
set reverse_map 0
# Parse command line
set l [llength $argv]
set i 0
set omits {}
while {$i < $l} {
    set arg [lindex $argv $i]
    switch -glob -- $arg {
        -v {
            incr verbose
        }
        -s {
            if {[string length $arg]} {
                set arg [lindex $argv [incr i]]
            }
            set trie(split) $arg
        }
        -p {
            if {[string length $arg]} {
                set arg [lindex $argv [incr i]]
            }
            set prefix $arg
        }
        -o {
            if {[string length $arg]} {
                set arg [lindex $argv [incr i]]
            }
            set ofile $arg
        }
        -O {
            if {[string length $arg]} {
                set arg [lindex $argv [incr i]]
            }
            lappend omits $arg
        }
        -r {
            set reverse_map 1
        }
        default {
            lappend ifiles $arg
        }
    }
    incr i
}
if {![info exists ifiles]} {
    puts "charconv.tcl: missing input file(s)"
    usage
}
 
set ofilehandle [open ${ofile}.tmp w]
preamble_trie $ofilehandle $ifiles $ofile
 
foreach ifile $ifiles {
    readfile $ifile $ofilehandle $prefix $omits $reverse_map
}
close $ofilehandle
 
file rename -force ${ofile}.tmp ${ofile}