tii

commit 8981e6cfc81a5e4bb10c19b908a9152634c53822
parent c026d357531e952367e19631d7859e86ccca45a7
Author: Luxferre <lux@ferre>
Date:   Mon, 21 Oct 2024 22:10:54 +0300

Added the cli echo viewer and rudimentary README

Diffstat:
A
README
 | 
134
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
M
tiifetch.tcl
 | 
38
++++++++++++++++++++++++++++++++------
A
tiiview.tcl
 | 
147
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

3 files changed, 313 insertions(+), 6 deletions(-)
diff --git a/README b/README
@@ -0,0 +1,134 @@
+tii: a Tcl-based suite for working with ii/IDEC protocol
+========================================================
+This suite implements the client side of ii and (partially) IDEC protocols of
+distributed, cross-platform, text-based communication (a FIDOnet successor, so
+to speak). Protocol documentation can be found here for both ii and IDEC:
+https://github.com/IDEC-net/new-docs/blob/master/protocol-en.md
+(there will be an effort to write a more concise version of that doc)
+
+The tii suite requires at least Tcl 8.6 to run. Running it inside starpacks is
+possible but not recommended.
+
+The tii repo consists of the following parts:
+
+* tiifetch.tcl: the core ii/IDEC message fetching library and CLI utility
+* tiipost.tcl: the core ii/IDEC message posting library and CLI utility
+* tiiview.tcl: the CLI viewer of the fetched ii/IDEC messages and conferences
+* tiidb: the (overridable) directory that contains all messages and echo lists
+* stations.txt: the list of stations to be auto-fetched by tiifetch when none
+  of its command-line parameters is passed
+* auth.txt: the list of station/authstring mappings to be used by tiipost when
+  posting messages to a particular station
+* tiix.tcl: the GUI ii/IDEC viewer that also leverages tiifetch and tiipost to
+  provide fetching and posting functionality
+
+Readiness status
+----------------
+* tiifetch.tcl: ready/tested
+* tiipost.tcl: planned
+* tiiview.tcl: ready/tested
+* tiidb (format): ready/tested
+* stations.txt (format): ready/tested
+* auth.txt (format): ready to be implemented
+* tiix.tcl: planned
+* Overall status: work in progress
+
+Usage
+-----
+This section is a work in progress and will be updated as more components are
+developed.
+
+### Fetching the messages (tiifetch.tcl): ###
+
+tclsh tiifetch.tcl [station_url] [echos] [dbdir]
+
+This command will fetch all messages into the dbdir ("tiidb" in the script dir
+by default) from the station_url (can be empty, see below) based on the echo
+conference names (can be delimited with slash /, comma (,) or semicolon (;))
+and create the corresponding file structure if it's missing.
+
+Fetching is supported for the following station URL schemes and protocols:
+
+* HTTP (http://)
+* HTTPS (https://)
+* Gopher (gopher://)
+* Gopher over TLS (gophers://)
+* Finger (finger://)
+* Nex (nex://)
+* Spartan (spartan://)
+* Gemini (gemini://)
+
+If the station_url parameter is empty or no parameters are passed at all,
+tiifetch.tcl will look for a file called stations.txt that lists (each on a
+new line) all the station URLs to sync from. Messages from all listed stations
+will be merged into the same echo conference database.
+
+### Viewing the messages from CLI (tiiview.tcl): ###
+
+tclsh tiiview.tcl [echo_name] [filter_string] [line_width] [dbdir]
+
+If the echo_name parameter is passed, this command will write all formatted
+messages from the coresponding echo conference to the standard output.
+You should pipe this message stream to your $PAGER terminal application, like
+less or most. The messages will be formatted according to the filter_string
+format (see below). If the filter string is empty or omitted, the messages
+will appear in the exact order their IDs appear in the echo conference file.
+
+If the echo_name parameter is empty or no parameters are passed at all, this
+command will output the list of echo conferences registered in the dbdir.
+The conference list will be alphabetically ordered and the filter_string will
+have no effect at all if you pass it.
+
+If the line_width parameter is omitted, the text reflows to 80 chars per line.
+
+If the dbdir is ommitted, it defaults to "tiidb" in the script directory.
+
+This component is fully offline and can only work with a compatible message
+database that tiifetch.tcl can generate (see "Message database format").
+
+The filter string can take one of the following basic forms:
+
+* h[number]: only take [number] messages from the head (start) of the list
+* t[number]: only take [number] messages from the tail (end) of the list
+* rh[number]: same as h but output messages from newest to oldest in the list
+* rt[number]: same as t but output messages from newest to oldest in the list
+
+If [number] is 0 then it means no message limit.
+The reverse operation is always done after limiting the results.
+
+e.g. rt50 will output 50 newest messages in the conference, starting from the
+most recent one. The default basic value is h0, so no filter applied will mean
+outputting all messages from the oldest to the newest.
+
+You can extend the basic form by appending a search regular expression to it,
+like this: [form]/[regex]. Note that the regex always applies AFTER the basic
+filtering has been done. Also note that the search is done within all fields
+of the message. E.g. h100/retro will find the retro-themed messages among the
+first 100 of them.
+
+Message database format
+-----------------------
+The tiidb format is based upon the official ii/IDEC developer recommendations
+and is fully plaintext-based, portable and very simple:
+
+* Message contents are stored decoded in the "msg/" subdirectory. The file
+  names are their plain 20-character hash IDs.
+* Echo contents are stored in the "echo/" subdirectory. The file names are the
+  conference names verbatim, containing newline-separated message IDs that
+  belong to those conferences, in order they were published there.
+  Every echo file ends with a blank line.
+
+FAQ
+---
+
+- Does tii implement any IDEC extensions?
+
+Only one: fetching list.txt from the station to get the entire list of echo
+conferences served by this station. This is something that the original ii
+spec did not support.
+
+
+Credits
+-------
+Created by Luxferre in 2024, released into public domain with no warranties.
+
diff --git a/tiifetch.tcl b/tiifetch.tcl
@@ -1,13 +1,12 @@
 #!/usr/bin/env tclsh
 # tiifetch: fetch all data from an ii/idec station into the local text db
 # (see https://github.com/idec-net/new-docs/blob/master/protocol-en.md)
-# Usage: tiifetch station_http_url [echos] [db_dir]
+# Usage: tiifetch.tcl [station_url] [echos] [db_dir]
 # The echo list should be delimited with slash (/), comma (,) or semicolon (;)
 # if no echos are specified (or "" is passed), then list.txt will be fetched
 # and then all missing echo content from it will be downloaded
 # If db_dir isn't specified, it's fetched and merged into the
-# tiidb directory in the program root
-# with echoconfs and messages respectively
+# tiidb directory in the program root with echoconfs and messages respectively
 # This component only fetches the messages, doesn't parse or display them
 # Supported protocols: HTTP, HTTPS, Gemini, Spartan, Gopher/Finger/Nex
 # Depends on Tcllib for URI parsing
@@ -268,6 +267,24 @@ proc fetchiidb {url echos dbdir dolog} {
   }
 }
 
+proc massfetch {echos dbdir dolog} {
+  global appdir
+  if {$dolog eq 1} {puts "No ii/idec station URL specified, using stations.txt"}
+  set stfile [file join $appdir "stations.txt"]
+  if {[file exists $stfile]} {
+    set stlist [split [readfile $stfile] "\n"]
+    foreach station $stlist {
+      set station [string trim $station]
+      if {$station ne ""} {
+        if {$dolog eq 1} {puts "Fetching from $station"}
+        fetchiidb $station $echos $dbdir $dolog
+      }
+    }
+  } else {
+    if {$dolog eq 1} {puts "No stations.txt found, bailing out!"}
+  }
+}
+
 # end of procs, start the entrypoint
 if {![info exists argv0] || [file tail [info script]] ne [file tail $argv0]} {return}
 
@@ -278,13 +295,22 @@ if [string match *app-tiifetch $appdir] {
   set appdir [file normalize [file join $appdir ".." ".." ".." ]]
 }
 
+set localdbdir [file join $appdir "tiidb"]
 if {$argc > 0} {
-  set localdbdir [file join $appdir "tiidb"]
   if {$argc > 2} {
     set localdbdir [lindex $argv 2]
   }
   puts "Fetching messages, please wait..."
-  fetchiidb [lindex $argv 0] [lindex $argv 1] $localdbdir 1
+  set sturl [string trim [lindex $argv 0]]
+  if {$sturl eq ""} {
+    massfetch [lindex $argv 1] $localdbdir 1
+  } else {
+    fetchiidb $sturl [lindex $argv 1] $localdbdir 1
+  }
   puts "Messages fetched"
-} else {puts "No ii/idec station URL specified!"}
+} else {
+  puts "Fetching messages, please wait..."
+  massfetch "" $localdbdir 1
+  puts "Messages fetched"
+}
 
diff --git a/tiiview.tcl b/tiiview.tcl
@@ -0,0 +1,147 @@
+#!/usr/bin/env tclsh
+# tiiview: view ii/idec messages from the local text db
+# Usage: tiiview.tcl [echo_name] [filter_string] [termwidth] [dbdir]
+# Created by Luxferre in 2024, released into public domain
+
+# file read helper
+proc readfile {fname} {
+  set fp [open $fname r]
+  fconfigure $fp -encoding utf-8
+  set data [read $fp]
+  close $fp
+  return $data
+}
+
+# basic text reflow helper
+# list in, string out
+proc tiiflow {lines width} {
+  set outtext {}
+  foreach inl $lines {
+    set l [string length $inl]
+    if {$l <= $width} {
+      append outtext "$inl\n"
+    } else {
+      set wordset ""
+      set words [split $inl " "]
+      foreach w $words {
+        set candidate "$wordset $w"
+        if {[string length $candidate] <= $width} {
+          set wordset $candidate
+        } else {
+          append outtext "$wordset\n"
+          set wordset $w
+        }
+      }
+      append outtext "$wordset\n"
+    }
+  }
+  return $outtext
+}
+
+# parse and pretty-print the found message
+proc formatmessage {msgdata msgid globalwidth} {
+  set globalline [string repeat = $globalwidth]
+  set hdrline [string repeat - $globalwidth]
+  set msglines [lmap s [split $msgdata "\n"] {string trimright $s}]
+  # parsing according to the spec, first 7 lines are:
+  # tags, echoarea, timestamp, msgfrom, msgfrom_addr, msgto, subj
+  # and then an empty line and the message body follows
+  set tags [split [lindex $msglines 0] "/"]
+  if {[dict exists $tags repto]} {
+    set replyto [dict get $tags repto]
+  } else {set replyto ""}
+  set echoarea [lindex $msglines 1]
+  set timestamp [lindex $msglines 2]
+  set msgfrom [lindex $msglines 3]
+  set msgfromaddr [lindex $msglines 4]
+  set msgto [lindex $msglines 5]
+  set subj [lindex $msglines 6]
+  set msgbody [tiiflow [lrange $msglines 8 end] $globalwidth]
+  set tz ""
+  set renderedts [clock format $timestamp -format {%Y-%m-%d %H:%M:%S} -timezone $tz]
+  catch { # because pipe can be broken anytime
+    puts "\[$renderedts\] ii://$msgid"
+    puts "$echoarea - $msgfrom ($msgfromaddr) to $msgto"
+    if {$replyto ne ""} {
+      puts "Replied to: ii://$replyto"
+    }
+    puts "Subj: $subj"
+    puts $hdrline
+    puts "$msgbody$globalline\n"
+  }
+}
+
+# entry point
+set scriptpath [file normalize [info script]]
+set appdir [file dirname $scriptpath]
+# check if we're running from a starpack
+if [string match *app-tiiview $appdir] {
+  set appdir [file normalize [file join $appdir ".." ".." ".." ]]
+}
+set localdbdir [file join $appdir "tiidb"]
+set echoname ""
+set filterstr ""
+set twidth 80
+if {$argc > 0} {
+  if {$argc > 1} {
+    set filterstr [string trim [lindex $argv 1]]
+  }
+  if {$argc > 2} {
+    set twidth [expr {int([lindex $argv 2])}]
+  }
+  if {$argc > 3} {
+    set localdbdir [lindex $argv 3]
+  }
+  set echoname [string trim [lindex $argv 0]]
+}
+if {$twidth < 20} {set twidth 80}
+if {$filterstr eq ""} {set filterstr "h0"}
+set msgdir [file join $localdbdir "msg"]
+set echodir [file join $localdbdir "echo"]
+if {$echoname eq ""} { # list the echodir
+  set echos [glob -tails -directory $echodir -nocomplain -types f "*.*"]
+  puts [join [lsort $echos] "\n"]
+} else { # fetch the actual contents
+  set echofile [file join $echodir $echoname]
+  if {[file exists $echofile]} {
+    set msglist [split [readfile $echofile] "\n"]
+    set filters [split $filterstr "/"]
+    set basicmod [string trim [lindex $filters 0]]
+    set filterregex {} 
+    if {[llength $filters] > 1} {
+      set filterregex [string trim [lindex $filters 1]]
+    }
+    set doreverse 0
+    if {[string first r $basicmod] > -1} {set doreverse 1}
+    set dotail 0
+    if {[string first t $basicmod] > -1} {set dotail 1}
+    set numitems 0
+    if {[regexp {\d+} $basicmod foundnum]} {set numitems $foundnum}
+    # perform the element filtering
+    if {$numitems > 0} {
+      incr numitems -1
+      if {$dotail eq 1} {
+        set msglist [lrange $msglist end-$numitems end]
+      } else {
+        set msglist [lrange $msglist 0 $numitems]
+      }
+    }
+    if {$doreverse eq 1} {
+      set msglist [lreverse $msglist]
+    }
+    foreach msgid $msglist { # iterate over the list after filtering
+      set msgid [string trim $msgid]
+      if {$msgid ne ""} {
+        set msgfile [file join $msgdir $msgid]
+        if {[file exists $msgfile]} {
+          set msgdata [readfile $msgfile]
+          set pass 1
+          if {$filterregex ne {}} {
+            set pass [regexp -line -nocase -- $filterregex $msgdata]
+          }
+          if {$pass eq 1} {formatmessage $msgdata $msgid $twidth}
+        }
+      }
+    }
+  } else {puts "This echo conference doesn't exist in the local DB!"}
+}