commit 203f6a30ca7b6661830f8d61c61d1cf755f35395
parent badc92ea99dd359e624a2a9bfb883edcd031cb92
Author: Luxferre <lux@ferre>
Date: Sat, 23 Nov 2024 08:11:48 +0200
implemented compression for HTTP-requested data
Diffstat:
| M | tiid.tcl | | | 153 | +++++++++++++++++++++++++++++++++++++++++++++++-------------------------------- |
| M | tiifetch.tcl | | | 1 | + |
2 files changed, 93 insertions(+), 61 deletions(-)
diff --git a/tiid.tcl b/tiid.tcl
@@ -25,6 +25,7 @@ set nodename "tiid"
proc createdb {fname} {
sqlite3 fdb $fname
fdb eval {
+ PRAGMA journal_mode=WAL;
CREATE TABLE `msg` (`id` INTEGER PRIMARY KEY AUTOINCREMENT,
`msgid` VARCHAR(20) NOT NULL UNIQUE,
`timestamp` INT NOT NULL,
@@ -256,12 +257,25 @@ proc indexpage {} {
# TCP logic here
+# compression function
+proc repcompress {compfunc data} {
+ if {$compfunc eq {gzip}} {
+ return [zlib gzip $data -level 9]
+ } else {
+ return [zlib $compfunc $data 9]
+ }
+}
+
# error report/reply
-proc reperror {sock ishttp errmsg} {
+proc reperror {sock ishttp errmsg compfunc} {
set errmsg "error: $errmsg\n"
if {$ishttp eq 1} {
set msglen [string length $errmsg]
set hdrs "Content-Type: text/plain;charset=utf-8\r\nContent-Length: $msglen\r\nConnection: close\r\n"
+ if {$compfunc ne {none}} {
+ set hdrs [string cat $hdrs "Content-Encoding: $compfunc\r\n"]
+ set errmsg [repcompress $compfunc $errmsg]
+ }
puts -nonewline $sock "HTTP/1.0 400 Bad Request\r\n$hdrs\r\n$errmsg"
} else {
puts -nonewline $sock "$errmsg"
@@ -270,10 +284,14 @@ proc reperror {sock ishttp errmsg} {
}
# successful reply with data
-proc repdata {sock ishttp data} {
+proc repdata {sock ishttp data compfunc} {
if {$ishttp eq 1} {
set msglen [string length $data]
set hdrs "Content-Type: text/plain;charset=utf-8\r\nContent-Length: $msglen\r\nConnection: close\r\n"
+ if {$compfunc ne {none}} {
+ set hdrs [string cat $hdrs "Content-Encoding: $compfunc\r\n"]
+ set data [repcompress $compfunc $data]
+ }
puts -nonewline $sock "HTTP/1.0 200 OK\r\n$hdrs\r\n$data"
} else {
puts -nonewline $sock $data
@@ -284,7 +302,7 @@ proc repdata {sock ishttp data} {
# path router
# it only must write to the socket, not read from it or close it
# supported paths: /e, /m, /u/e, /u/m, /u/point, /list.txt, /blacklist.txt
-proc routepath {dbfile sock ishttp path body} {
+proc routepath {dbfile sock ishttp path body compfunc} {
fconfigure $sock -translation binary
set pathparts [split [string trim $path] /]
if {[llength $pathparts] > 1} {
@@ -309,12 +327,12 @@ proc routepath {dbfile sock ishttp path body} {
[validecho $ename] ? $ename : [continue]
}}]
if {[llength $erange] > 0} { # recheck length after validation
- repdata $sock $ishttp [indexechos $dbfile $erange 1 $offset $limit]
+ repdata $sock $ishttp [indexechos $dbfile $erange 1 $offset $limit] $compfunc
} else {
- reperror $sock $ishttp "invalid request"
+ reperror $sock $ishttp "invalid request" $compfunc
}
} else {
- reperror $sock $ishttp "invalid request"
+ reperror $sock $ishttp "invalid request" $compfunc
}
}
m { # validate and shape the message ID list
@@ -322,9 +340,9 @@ proc routepath {dbfile sock ishttp path body} {
[validmsgid $mid] ? $mid : [continue]
}}]
if {[llength $mrange] > 0} { # we have some valid messages
- repdata $sock $ishttp [multimsg $dbfile $mrange]
+ repdata $sock $ishttp [multimsg $dbfile $mrange] $compfunc
} else {
- reperror $sock $ishttp "invalid request"
+ reperror $sock $ishttp "invalid request" $compfunc
}
}
point {
@@ -349,12 +367,12 @@ proc routepath {dbfile sock ishttp path body} {
if {$authstr ne "" && $msgbody ne ""} {
set postres [postmsg $dbfile $authstr $msgbody]
if [string match "msg ok*" $postres] {
- repdata $sock $ishttp $postres
+ repdata $sock $ishttp $postres $compfunc
} else {
- reperror $sock $ishttp $postres
+ reperror $sock $ishttp $postres $compfunc
}
} else {
- reperror $sock $ishttp "invalid request"
+ reperror $sock $ishttp "invalid request" $compfunc
}
}
push {
@@ -369,68 +387,68 @@ proc routepath {dbfile sock ishttp path body} {
set msgbody [decurl [dict get $params upush]]
}
} else {
- reperror $sock $ishttp "/u/push is only available over HTTP POST"
+ reperror $sock $ishttp "/u/push is only available over HTTP POST" $compfunc
return
}
if {$authstr ne "" && $msgbody ne ""} {
set postres [postbundle $dbfile $authstr $msgbody]
if [string match "message saved: ok*" $postres] {
- repdata $sock $ishttp $postres
+ repdata $sock $ishttp $postres $compfunc
} else {
- reperror $sock $ishttp $postres
+ reperror $sock $ishttp $postres $compfunc
}
} else {
- reperror $sock $ishttp "invalid request"
+ reperror $sock $ishttp "invalid request" $compfunc
}
}
default {
- reperror $sock $ishttp "invalid request"
+ reperror $sock $ishttp "invalid request" $compfunc
}
}
} else {
- reperror $sock $ishttp "invalid request"
+ reperror $sock $ishttp "invalid request" $compfunc
}
}
e {
if {[llength $pathparts] > 2} {
set echoname [string trim [lindex $pathparts 2]]
if {[validecho $echoname]} {
- repdata $sock $ishttp [indexechos $dbfile [list $echoname] 0 0 0]
+ repdata $sock $ishttp [indexechos $dbfile [list $echoname] 0 0 0] $compfunc
} else {
- reperror $sock $ishttp "invalid request"
+ reperror $sock $ishttp "invalid request" $compfunc
}
} else {
- reperror $sock $ishttp "invalid request"
+ reperror $sock $ishttp "invalid request" $compfunc
}
}
m {
if {[llength $pathparts] > 2} {
set mid [string trim [lindex $pathparts 2]]
if {[validmsgid $mid]} {
- repdata $sock $ishttp [singlemsg $dbfile $mid]
+ repdata $sock $ishttp [singlemsg $dbfile $mid] $compfunc
} else {
- reperror $sock $ishttp "invalid request"
+ reperror $sock $ishttp "invalid request" $compfunc
}
} else {
- reperror $sock $ishttp "invalid request"
+ reperror $sock $ishttp "invalid request" $compfunc
}
}
list.txt {
- repdata $sock $ishttp [listechos $dbfile]
+ repdata $sock $ishttp [listechos $dbfile] $compfunc
}
blacklist.txt {
- repdata $sock $ishttp [blacklisted $dbfile]
+ repdata $sock $ishttp [blacklisted $dbfile] $compfunc
}
{} {
- repdata $sock $ishttp [indexpage]
+ repdata $sock $ishttp [indexpage] $compfunc
}
default {
- reperror $sock $ishttp "invalid request"
+ reperror $sock $ishttp "invalid request" $compfunc
}
}
} else {
- reperror $sock $ishttp "invalid request"
+ reperror $sock $ishttp "invalid request" $compfunc
}
}
@@ -440,43 +458,56 @@ proc reqhandler {dbfile sock} {
# ignore all invalid requests by closing the connection
if {[gets $sock line] >= 0} {
if {[string match /* $line]} { # bare TCP request (Gopher/Nex)
- routepath $dbfile $sock 0 [string trim $line] {}
- } elseif {[string match -nocase {GET /*} $line]} { # HTTP GET request
- set rparts [split [string trimleft $line]]
- if {[llength $rparts] > 1} { # valid GET request
- routepath $dbfile $sock 1 [lindex $rparts 1] {}
- }
- } elseif {[string match -nocase {POST /*} $line]} { # HTTP POST request
- set rparts [split [string trimleft $line]]
- if {[llength $rparts] > 1} { # valid POST request, read POST headers and data
- set hdrread 0
- set hdata {}
- set pdata {}
- while {$hdrread < 1} {
- if {[eof $sock] || [catch {gets $sock line}]} {
- break
- } else {
- if {$line eq ""} {
- incr hdrread
- continue
- }
- set hparts [split [string trimleft $line] :]
- if {[llength $hparts] > 1} {
- set hname [string tolower [string trimright [lindex $hparts 0]]]
- set hval [string trim [lindex $hparts 1]]
- dict set hdata $hname $hval
- }
+ routepath $dbfile $sock 0 [string trim $line] {} none
+ } else { # HTTP request, read headers
+ set hdrread 0
+ set hdata {}
+ set docomp none
+ while {$hdrread < 1} {
+ if {[eof $sock] || [catch {gets $sock hline}]} {
+ break
+ } else {
+ if {$hline eq ""} {
+ incr hdrread
+ continue
+ }
+ set hparts [split [string trimleft $hline] :]
+ if {[llength $hparts] > 1} {
+ set hname [string tolower [string trimright [lindex $hparts 0]]]
+ set hval [string trim [lindex $hparts 1]]
+ dict set hdata $hname $hval
}
}
- set readlen 0
- if {[dict exists $hdata content-length]} {
- set readlen [dict get $hdata content-length]
+ }
+ if {[dict exists $hdata accept-encoding]} { # detect if we can gzip the output
+ set enclist [dict get $hdata accept-encoding]
+ if {[string match -nocase {*gzip*} $enclist]} {
+ set docomp gzip
+ } elseif {[string match -nocase {*deflate*} $enclist]} {
+ set docomp deflate
+ } elseif {[string match -nocase {*compress*} $enclist]} {
+ set docomp compress
}
- if {$readlen > 0} { # read the data defined by content-length header
- fconfigure $sock -translation {binary lf} -buffering none
- set pdata [read $sock $readlen]
+ }
+ if {[string match -nocase {GET /*} $line]} { # GET request
+ set rparts [split [string trimleft $line]]
+ if {[llength $rparts] > 1} { # valid GET request
+ routepath $dbfile $sock 1 [lindex $rparts 1] {} $docomp
+ }
+ } elseif {[string match -nocase {POST /*} $line]} { # POST request
+ set rparts [split [string trimleft $line]]
+ if {[llength $rparts] > 1} { # valid POST request, read POST headers and data
+ set pdata {}
+ set readlen 0
+ if {[dict exists $hdata content-length]} {
+ set readlen [dict get $hdata content-length]
+ }
+ if {$readlen > 0} { # read the data defined by content-length header
+ fconfigure $sock -translation {binary lf} -buffering none
+ set pdata [read $sock $readlen]
+ }
+ routepath $dbfile $sock 1 [lindex $rparts 1] [string trimleft $pdata] $docomp
}
- routepath $dbfile $sock 1 [lindex $rparts 1] [string trimleft $pdata]
}
}
}
diff --git a/tiifetch.tcl b/tiifetch.tcl
@@ -203,6 +203,7 @@ proc n2p_id {binmsg} {
proc createdb {fname} {
sqlite3 fdb $fname
fdb eval {
+ PRAGMA journal_mode=WAL;
CREATE TABLE `msg` (`id` INTEGER PRIMARY KEY AUTOINCREMENT,
`msgid` VARCHAR(20) NOT NULL UNIQUE,
`timestamp` INT NOT NULL,
