squash merge cfg-update

README.md

@@ -5,11 +5,16 @@ Simple and stylish text-to-html microblog generator.
 
 ## Requirements
 
-    python3 dateutil toml curl pycurl make urllib
+The following python modules are used within the repository.
 
-* `dateutil`, `toml`, `pycurl` are Python modules. 
+    toml tomlkit python_dateutil pycurl
-* `make` (optional), method for invoking the script. 
+
-* `urllib` (optional), for uploading multiple files to neocities (`neouploader.py`).
+* `tomlkit` (optional), for maintaining the configuration file between updates (`check-settings.py`).
+
+Some Gnu core utilities are expected to be present but can be substituted for other means.
+
+* `make` (optional), to invoke the script using Makefiles
+* `date` (optional), to generate timestamps when writing posts
 
 ## Usage
 
@@ -24,7 +29,7 @@ Using `make` is uptional; it does the following within a new directory:
     cp example/timeline.css ./timeline.css
     cp example/default.tpl ./template.tpl
     cp example/demo.txt ./content.txt
-    python microblog.py ./template.tpl ./content.txt > result.html
+    python src/microblog.py ./template.tpl ./content.txt > result.html
 
 This script generate a text file after operation.
 
@@ -64,11 +69,13 @@ Configuration options as understood by the script are tentative and may change i
 
 >This script is throwing KeyError after I ran git pull
 
-In most cases, this means I added new configuration options. You can resolve this error by copying and pasting the missing keys from `example/settings.toml` to `settings.toml`.
+In most cases, this means I added new configuration options. You can resolve this error by adding missing keys from `example/settings.toml` to `settings.toml`.
 
-The following command shows differences between the files.
+The following command can check for missing keys and update if needed.
 
-    diff settings.toml example/settings.toml
+    python src/check-settings.py
+
+Missing keys if any are initialized to default values from `example/settings.toml`.
 
 ## Anything else
 

example/Makefile

@@ -1,8 +1,12 @@
 
 
 all: demo tpl css settings
-	python microblog.py ./template.tpl ./content.txt > result.html
+	python src/microblog.py ./template.tpl ./content.txt > result.html
 
+check:
+	python src/check-settings.py
+
+# first time run only
 tpl:
 	cp ./example/default.tpl ./template.tpl
 

example/settings.toml

@@ -4,6 +4,11 @@ latestpages=["meta.json", "result.html"]
 [page]
 postsperpage = 20
 relative_css=["./style.css", "./timeline.css"]
+# this would be "latest.html" in earlier versions i.e
+# user.domain.tld/microblog/tags/tagname/latest.html
+# naming it as index enables paths like so
+# user.domain.tld/microblog/tags/tagname
+landing_page="index.html"
 
 [post]
 accepted_images= ["jpg", "JPG", "png", "PNG"]
@@ -11,8 +16,7 @@ accepted_images= ["jpg", "JPG", "png", "PNG"]
 tag_paragraphs=true
 # apply <p> tags even if a line contains the following
 inline_tags = ["i", "em", "b", "strong","u", "s", "a", "span"]
-# adds <br> or user defined string between each line
+date_format="%d %B %Y"
-# line_separator="<br>"
 format="""
 <div class="postcell" id="{__num__}">
     <div class="timestamp">{__timestamp__}
@@ -24,6 +28,11 @@ format="""
 """
 
 [post.buttons]
+format="""
+    <a class="buttons" href="{__url__}">{__label__}</a>
+"""
+
+[post.buttons.links]
 reply = "mailto:user@host.tld"
 test  = "https://toml.io/en/v1.0.0#array-of-tables"
 interact = "https://yoursite.tld/cgi?postid="
@@ -44,8 +53,8 @@ short-bio= "Your self-description. Anything longer than 150 characters is trunca
 
 [webring.following]
 list= ["https://likho.neocities.org/microblog/meta.json"]
+date_format = "%Y %b %d"
 format="""
-<div class="fill">
 <div class="postcell">
     <img src="{__avatar__}" alt="Avatar" class="avatar">
     <span class="wrapper"">
@@ -57,7 +66,6 @@ format="""
     </span>
     <p class="short-bio">{__shortbio__}</p>
 </div>
-</div>
 """
 
 # internally link avatars - avoids hotlinks

requirements.txt

@@ -0,0 +1,13 @@
+pycurl
+# ==7.45.3
+# pycurl==7.45.2
+
+python_dateutil
+# ==2.9.0.post0
+# python_dateutil==2.8.2
+
+toml
+# ==0.10.2
+
+tomlkit
+# ==0.12.5

src/check-settings.py

@@ -0,0 +1,134 @@
+
+import os, argparse
+from tomlkit import loads
+from tomlkit import dump
+
+def nest_dictionary(d, keys, val):
+    for key in keys:
+        d = d.setdefault(key, val)
+    return d
+
+class MicroblogConfig:
+    def __init__(self, given_config):
+        self.is_outdated = False
+        self.updated = given_config
+
+    def compare(self, sref, suser, keylist=[]):
+        # subtable of ref, subtable of user
+        updated = self.updated
+        # nnavigate to table
+        if keylist != []:
+            for key in keylist:
+                sref = sref[key]
+            for key in keylist:
+                suser = suser[key]
+            for key in keylist:
+                updated = updated[key]
+        for key in sref:
+            if key not in suser:
+                self.is_outdated = True
+                updated[key] =sref[key]
+                print("noticed '", key, "' missing from ", keylist)
+        nest_dictionary(self.updated, keylist, updated)
+        return
+
+    def check(self, r, u): # (reference, user)
+        for key in r:
+            if key == "latestpages": continue;
+            # post and webring have subtables
+            #   webring.profile
+            #   webring.following
+            #   webring.following.internal-avatars
+            #   post.gallery
+            #   post.buttons
+            try:
+                self.compare(r, u, [key])
+            except KeyError:
+                u[key] = dict()
+                print("missing top-level table '", key, '\'')
+                self.compare(r, u, [key])
+            if key == "webring":
+                self.compare(r, u, ["webring", "profile"])
+                self.compare(r, u, ["webring", "following"])
+                self.compare(r, u, ["webring", "following", "internal-avatars"])
+            if key == "post":
+                self.compare(r, u, ["post", "gallery"])
+                self.compare(r, u, ["post", "buttons"])
+        pass
+
+def load_files(user_conf_file):
+    script_dir = os.path.dirname(
+        os.path.abspath(__file__))
+    parent_dir = os.path.abspath(
+        os.path.join(script_dir, os.pardir))
+    target_folder = "example"
+    example = os.path.abspath(
+        os.path.join(parent_dir, target_folder))
+    ref_file = "%s/%s" % (example, "/settings.toml")
+    if not os.path.exists(ref_file):
+        return
+    ref_conf = dict()
+    with open(ref_file, 'r') as f:
+        ref_conf = loads(f.read())
+    user_conf = dict()
+    with open(user_conf_file, 'r') as f:
+        user_conf = loads(f.read())
+    return ref_conf, user_conf
+
+def multi_prompt(message):
+    try:
+        while True:
+            user_input = int(input(f"{message}").lower())
+            if  user_input < 3:
+                return user_input
+            else:
+                return 0
+    except KeyboardInterrupt:
+        print()
+    except ValueError:
+        pass
+    return 0
+
+def get_args():
+    p = argparse.ArgumentParser()
+    p.add_argument("--no-prompt", action="store_true", \
+        help="does not ask what to do if missing keys are detected")
+    p.add_argument("-c", "--check", type=str,\
+        help="sets/changes the file to be checked (default: settings.toml)")
+    args = p.parse_args()
+    if args.no_prompt:
+        print("'--no-prompt' set")
+    if args.check:
+        print("--check set", args.check)
+    return args.no_prompt, args.check
+
+def main(is_no_prompt, user_conf_file="settings.toml"):
+    print("checking ", user_conf_file)
+    reference, user_edited = load_files(user_conf_file)
+    mcfg = MicroblogConfig(user_edited)
+    mcfg.check(reference, user_edited)
+    if mcfg.is_outdated == False:
+        print("Your settings file is OK!")
+        return
+    message = """
+    Your settings file is outdated.
+    Do you want to...
+    \t 1. save new settings to new file
+    \t 2. update/overwrite existing settings
+    \t *. do nothing
+    """
+    response = 0 if is_no_prompt else multi_prompt(message)
+    out_file = str()
+    if response == 0:
+        return
+    elif response == 1:
+        out_file = "new.toml"
+    elif response == 2:
+        out_file = user_conf_file
+    with open(out_file, 'w') as f:
+        dump(mcfg.updated, f)
+    print("Wrote updated config to ", out_file)
+    pass
+
+if __name__ == "__main__":
+    main(*get_args())

src/microblog.py

@@ -3,17 +3,15 @@ import sys, os, traceback
 import dateutil.parser
 from time import strftime, localtime
 
-# returns html-formatted string
+def make_buttons(btn_conf, msg_id):
-def make_buttons(btn_dict, msg_id):
+    fmt = btn_conf["format"]
-    buttons = "<div class=\"buttons\">"
+    buttons = str()
-    fmt = "<a href=\"%s\">[%s]</a>"
+    for key in btn_conf["links"]:
-    for key in btn_dict:
+        url =  btn_conf["links"][key]
-        url = btn_dict[key] 
         if url[-1] == '=':
-        # then interpret it as a query string
             url += str(msg_id)
-        buttons += fmt % (url,key)
+        buttons += fmt.format(
-    buttons += "</div>"
+            __url__=url, __label__ = key)
     return buttons
 
 # apply div classes for use with .css
@@ -132,8 +130,6 @@ def markup(message, config):
         ignore = config["inline_tags"]
     parser = My_Html_Parser(ignore)
     sep = ""
-    if "line_separator" in config:
-        sep = config["line_separator"]
     for line in message:
         images = [] # list of integers
         parser.feed(line)
@@ -172,9 +168,11 @@ class Post:
         return int(t.timestamp())
 
     # format used for display
-    def get_short_time(self):
+    def get_short_time(self, form):
+        if form == "":
+            form = "%y %b %d"
         t = dateutil.parser.parse(self.timestamp)
-        return t.strftime("%y %b %d")
+        return t.strftime(form)
 
 def parse_txt(filename):
     content   = []
@@ -203,7 +201,7 @@ def parse_txt(filename):
                 state = 0 
     return posts
 
-def get_posts(posts, config):
+def get_posts(posts, config, newest = None):
     taginfos = []
     tagcloud = dict() # (tag, count)
     tagged   = dict() # (tag, index of message)
@@ -211,21 +209,29 @@ def get_posts(posts, config):
     count = total
     index = count # - 1
     timeline = []
-    btns = None
+    df = ""
+    subset = []
+    if "date_format" in config:
+        df = config["date_format"]
     for post in posts:
         markedup, tags = markup(post.message, config)
         count -= 1
         index -= 1
         timeline.append(
-            make_post(count, post.get_short_time(), config, markedup)
+            make_post(count, post.get_short_time(df), config, markedup)
         )
         for tag in tags:
             if tagcloud.get(tag) == None:
                 tagcloud[tag] = 0
             tagcloud[tag] += 1
+            if newest is not None and (total - (1 + count)) < newest:
+                subset.append(tag)
+            if newest is None \
+            or newest is not None and tag in subset:
                 if tagged.get(tag) == None:
                     tagged[tag] = []
                 tagged[tag].append(index)
+    # print(tagged, file=sys.stderr)
     return timeline, tagcloud, tagged
 
 def make_tagcloud(d, rell):
@@ -282,16 +288,14 @@ class Paginator:
 
     def paginate(self, template, tagcloud, timeline, is_tagline=False):
         outfile = "%s/%s" % (self.SUBDIR, self.FILENAME)
-        timeline.reverse() # reorder from oldest to newest
+        l = len(timeline)
-        start = 0
+        for i in range(0, self.TOTAL_PAGES):
-        for i in range(start, self.TOTAL_PAGES):
             fn = outfile % i
             with open(fn, 'w') as f:
                 self.written.append(fn)
-                prev = self.PPP * i
+                prev = l - (self.PPP * i)
-                curr = self.PPP * (i+1)
+                curr = l - self.PPP * (i+1)
-                sliced = timeline[prev:curr]
+                sliced = timeline[curr:prev]
-                sliced.reverse()
                 f.write(self.singlepage(template, tagcloud, sliced, i, "."))
         return
 
@@ -315,19 +319,20 @@ if __name__ == "__main__":
         p = argparse.ArgumentParser()
         p.add_argument("template", help="an html template file")
         p.add_argument("content", help="text file for microblog content")
-        p.add_argument("--sort", \
+        p.add_argument("--sort", action="store_true", \
             help="sorts content from oldest to newest"
-                " (this is a separate operation from page generation)", \
+                " (this is a separate operation from page generation)")
-            action="store_true")
+        p.add_argument("--skip-fetch", action="store_true", \
-        p.add_argument("--skip-fetch", \
             help="skips fetching profile data from remote sources;"
-                 " has no effect if webring is not enabled",\
+                 " has no effect if webring is not enabled")
-            action="store_true")
+        p.add_argument("--new-posts", type=int, nargs='?',
+            help="generate pages based only on new entries; " \
+                "if I wrote 5 new posts then --new-posts=5'")
         args = p.parse_args()
         if args.sort:
             sort(args.content)
             exit()
-        return args.template, args.content, args.skip_fetch
+        return args.template, args.content, args.skip_fetch, args.new_posts
 
     # assume relative path
     def demote_css(template, css_list,  level=1):
@@ -342,7 +347,7 @@ if __name__ == "__main__":
             tpl = tpl.replace(css, ("%s%s" % (prepend, css) ))
         return tpl
 
-    def writepage(template, timeline, tagcloud, config, subdir = None):
+    def writepage(template, timeline, tagcloud, config, subdir = None, paginate = True):
         count  = len(timeline)
         html   = ""
         with open(template,'r') as f:
@@ -356,10 +361,11 @@ if __name__ == "__main__":
         except Exception as e:
             print("error: ",e, ("(number of posts = %i)" % count), file=sys.stderr)
             exit()
+        index = config["landing_page"]
         latest = timeline[:pagectrl.PPP]
-        link_from_top    = "./tags/%s/latest.html"
+        link_from_top    = "./tags/%s/"  + index
-        link_from_subdir = "../tags/%s/latest.html"
+        link_from_subdir = "../tags/%s/" + index
-        link_from_tagdir = "../%s/latest.html"
+        link_from_tagdir = "../%s/"      + index
         cloud = ""
         level = 1
         is_tagline = False
@@ -375,20 +381,20 @@ if __name__ == "__main__":
             else:
                 cloud    = make_tagcloud(tagcloud, link_from_subdir)
             demoted  = demote_css(html, config["relative_css"], level)
-            filename = "%s/latest.html" % subdir
+            filename = "%s/%s" % (subdir, index)
             with open(filename, 'w') as f: # landing page for tag
                 pagectrl.written.append(filename)
                 page = pagectrl.singlepage(demoted, cloud, latest, p=".")
                 f.write(page)
+        if paginate:
             pagectrl.paginate(
                 demote_css(html, config["relative_css"], level), 
                 cloud, timeline, is_tagline)
         return pagectrl.written
 
     import toml
-    def load_settings():
+    def load_settings(filename = "settings.toml"):
         s = dict()
-        filename = "settings.toml"
         if os.path.exists(filename):
             with open(filename, 'r') as f:
                 s = toml.loads(f.read())
@@ -457,7 +463,7 @@ if __name__ == "__main__":
                     print(e)
             return json_objs
 
-        def render(profiles, template):
+        def render(profiles, template, date_format):
             rendered = []
             SHORT_BIO_LIMIT = 150
             for profile in profiles:
@@ -478,7 +484,7 @@ if __name__ == "__main__":
                         __post_count__ = post_count,
                         __shortbio__= escape(self_desc),
                         __lastupdated__= strftime(
-                            "%Y %b %d", localtime(epoch_timestamp)) )
+                            date_format, localtime(epoch_timestamp)) )
                     rendered.append(foo)
                 except KeyError as e:
                     print("remote profile is missing key: ", e, file=sys.stderr)
@@ -519,10 +525,9 @@ if __name__ == "__main__":
         try:
             list_of_json_objs.sort(key=lambda e: e["last-updated"], reverse=True)
         except KeyError: pass
-        return render(list_of_json_objs, f_cfg["format"])
+        return render(list_of_json_objs, f_cfg["format"], f_cfg["date_format"])
 
-    def main():
+    def main(tpl, content, skip_fetch, new_posts):
-        tpl, content, skip_fetch = get_args()
         cfg = load_settings()
         if cfg == None:
             print("exit: no settings.toml found.", file=sys.stderr)
@@ -534,25 +539,31 @@ if __name__ == "__main__":
             print("exit: table 'page' absent in settings.toml", file=sys.stderr)
             return
         p = parse_txt(content)
-        tl, tc, tg = get_posts(p, cfg["post"])
+        tl, tc, tg = get_posts(p, cfg["post"], new_posts)
         if tl == []:
             return
         # main timeline
         updated = []
-        updated += writepage(tpl, tl, tc, cfg["page"])
+        updated += writepage(tpl, tl, tc, cfg["page"], 
+            paginate=True if new_posts is None else False)
         # timeline per tag
         if tc != dict() and tg != dict():
             if not os.path.exists("tags"):
                 os.mkdir("tags")
+        tl.reverse()
         for key in tg.keys():
             tagline = []
             for index in tg[key]:
                 tagline.append(tl[index])
             # [1:] means to omit hashtag from dir name
+            wp = True # will paginate
+            if new_posts is not None \
+            and len(tagline) > cfg["page"]["postsperpage"]:
+                wp = False
             updated += writepage(
                 tpl, tagline, tc, cfg["page"], \
-                subdir="tags/%s" % key[1:]     \
+                subdir="tags/%s" % key[1:], \
-            ) 
+                paginate=wp) 
         if "webring" in cfg:
             if cfg["webring"]["enabled"] == True:
                 export_profile(
@@ -565,13 +576,11 @@ if __name__ == "__main__":
         with open("updatedfiles.txt", 'w') as f:
             for filename in updated:
                 print(filename, file=f) # sys.stderr)
-            if "latestpage" in cfg:
-                print(cfg["latestpage"], file=f)
             if "latestpages" in cfg:
                 for page in cfg["latestpages"]:
                     print(page, file=f)
     try:
-        main()
+        main(*get_args())
     except KeyError as e:
         traceback.print_exc()
         print("\n\tA key may be missing from your settings file.", file=sys.stderr)