diff --git a/README.md b/README.md index dc430c7..c4a60ea 100644 --- a/README.md +++ b/README.md @@ -5,11 +5,16 @@ Simple and stylish text-to-html microblog generator. ## Requirements - python3 dateutil toml curl pycurl make urllib +The following python modules are used within the repository. -* `dateutil`, `toml`, `pycurl` are Python modules. -* `make` (optional), method for invoking the script. -* `urllib` (optional), for uploading multiple files to neocities (`neouploader.py`). + toml tomlkit python_dateutil pycurl + +* `tomlkit` (optional), for maintaining the configuration file between updates (`check-settings.py`). + +Some Gnu core utilities are expected to be present but can be substituted for other means. + +* `make` (optional), to invoke the script using Makefiles +* `date` (optional), to generate timestamps when writing posts ## Usage @@ -24,7 +29,7 @@ Using `make` is uptional; it does the following within a new directory: cp example/timeline.css ./timeline.css cp example/default.tpl ./template.tpl cp example/demo.txt ./content.txt - python microblog.py ./template.tpl ./content.txt > result.html + python src/microblog.py ./template.tpl ./content.txt > result.html This script generate a text file after operation. @@ -64,11 +69,13 @@ Configuration options as understood by the script are tentative and may change i >This script is throwing KeyError after I ran git pull -In most cases, this means I added new configuration options. You can resolve this error by copying and pasting the missing keys from `example/settings.toml` to `settings.toml`. +In most cases, this means I added new configuration options. You can resolve this error by adding missing keys from `example/settings.toml` to `settings.toml`. -The following command shows differences between the files. +The following command can check for missing keys and update if needed. - diff settings.toml example/settings.toml + python src/check-settings.py + +Missing keys if any are initialized to default values from `example/settings.toml`. ## Anything else diff --git a/example/Makefile b/example/Makefile index 8a91908..1775778 100644 --- a/example/Makefile +++ b/example/Makefile @@ -1,8 +1,12 @@ all: demo tpl css settings - python microblog.py ./template.tpl ./content.txt > result.html + python src/microblog.py ./template.tpl ./content.txt > result.html +check: + python src/check-settings.py + +# first time run only tpl: cp ./example/default.tpl ./template.tpl diff --git a/example/settings.toml b/example/settings.toml index 8d3ad6d..2e6a229 100644 --- a/example/settings.toml +++ b/example/settings.toml @@ -4,6 +4,11 @@ latestpages=["meta.json", "result.html"] [page] postsperpage = 20 relative_css=["./style.css", "./timeline.css"] +# this would be "latest.html" in earlier versions i.e +# user.domain.tld/microblog/tags/tagname/latest.html +# naming it as index enables paths like so +# user.domain.tld/microblog/tags/tagname +landing_page="index.html" [post] accepted_images= ["jpg", "JPG", "png", "PNG"] @@ -11,8 +16,7 @@ accepted_images= ["jpg", "JPG", "png", "PNG"] tag_paragraphs=true # apply

tags even if a line contains the following inline_tags = ["i", "em", "b", "strong","u", "s", "a", "span"] -# adds
or user defined string between each line -# line_separator="
" +date_format="%d %B %Y" format="""

{__timestamp__} @@ -24,6 +28,11 @@ format=""" """ [post.buttons] +format=""" + {__label__} +""" + +[post.buttons.links] reply = "mailto:user@host.tld" test = "https://toml.io/en/v1.0.0#array-of-tables" interact = "https://yoursite.tld/cgi?postid=" @@ -44,19 +53,18 @@ short-bio= "Your self-description. Anything longer than 150 characters is trunca [webring.following] list= ["https://likho.neocities.org/microblog/meta.json"] +date_format = "%Y %b %d" format=""" -
-
- Avatar - -
- {__handle__} -
-
Last Update: {__lastupdated__}
- Posts: {__post_count__} - -

{__shortbio__}

-
+
+ Avatar + +
+ {__handle__} +
+
Last Update: {__lastupdated__}
+ Posts: {__post_count__} + +

{__shortbio__}

""" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..d9f8e90 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,13 @@ +pycurl +# ==7.45.3 +# pycurl==7.45.2 + +python_dateutil +# ==2.9.0.post0 +# python_dateutil==2.8.2 + +toml +# ==0.10.2 + +tomlkit +# ==0.12.5 diff --git a/src/check-settings.py b/src/check-settings.py new file mode 100644 index 0000000..6a308c3 --- /dev/null +++ b/src/check-settings.py @@ -0,0 +1,134 @@ + +import os, argparse +from tomlkit import loads +from tomlkit import dump + +def nest_dictionary(d, keys, val): + for key in keys: + d = d.setdefault(key, val) + return d + +class MicroblogConfig: + def __init__(self, given_config): + self.is_outdated = False + self.updated = given_config + + def compare(self, sref, suser, keylist=[]): + # subtable of ref, subtable of user + updated = self.updated + # nnavigate to table + if keylist != []: + for key in keylist: + sref = sref[key] + for key in keylist: + suser = suser[key] + for key in keylist: + updated = updated[key] + for key in sref: + if key not in suser: + self.is_outdated = True + updated[key] =sref[key] + print("noticed '", key, "' missing from ", keylist) + nest_dictionary(self.updated, keylist, updated) + return + + def check(self, r, u): # (reference, user) + for key in r: + if key == "latestpages": continue; + # post and webring have subtables + # webring.profile + # webring.following + # webring.following.internal-avatars + # post.gallery + # post.buttons + try: + self.compare(r, u, [key]) + except KeyError: + u[key] = dict() + print("missing top-level table '", key, '\'') + self.compare(r, u, [key]) + if key == "webring": + self.compare(r, u, ["webring", "profile"]) + self.compare(r, u, ["webring", "following"]) + self.compare(r, u, ["webring", "following", "internal-avatars"]) + if key == "post": + self.compare(r, u, ["post", "gallery"]) + self.compare(r, u, ["post", "buttons"]) + pass + +def load_files(user_conf_file): + script_dir = os.path.dirname( + os.path.abspath(__file__)) + parent_dir = os.path.abspath( + os.path.join(script_dir, os.pardir)) + target_folder = "example" + example = os.path.abspath( + os.path.join(parent_dir, target_folder)) + ref_file = "%s/%s" % (example, "/settings.toml") + if not os.path.exists(ref_file): + return + ref_conf = dict() + with open(ref_file, 'r') as f: + ref_conf = loads(f.read()) + user_conf = dict() + with open(user_conf_file, 'r') as f: + user_conf = loads(f.read()) + return ref_conf, user_conf + +def multi_prompt(message): + try: + while True: + user_input = int(input(f"{message}").lower()) + if user_input < 3: + return user_input + else: + return 0 + except KeyboardInterrupt: + print() + except ValueError: + pass + return 0 + +def get_args(): + p = argparse.ArgumentParser() + p.add_argument("--no-prompt", action="store_true", \ + help="does not ask what to do if missing keys are detected") + p.add_argument("-c", "--check", type=str,\ + help="sets/changes the file to be checked (default: settings.toml)") + args = p.parse_args() + if args.no_prompt: + print("'--no-prompt' set") + if args.check: + print("--check set", args.check) + return args.no_prompt, args.check + +def main(is_no_prompt, user_conf_file="settings.toml"): + print("checking ", user_conf_file) + reference, user_edited = load_files(user_conf_file) + mcfg = MicroblogConfig(user_edited) + mcfg.check(reference, user_edited) + if mcfg.is_outdated == False: + print("Your settings file is OK!") + return + message = """ + Your settings file is outdated. + Do you want to... + \t 1. save new settings to new file + \t 2. update/overwrite existing settings + \t *. do nothing + """ + response = 0 if is_no_prompt else multi_prompt(message) + out_file = str() + if response == 0: + return + elif response == 1: + out_file = "new.toml" + elif response == 2: + out_file = user_conf_file + with open(out_file, 'w') as f: + dump(mcfg.updated, f) + print("Wrote updated config to ", out_file) + pass + +if __name__ == "__main__": + main(*get_args()) diff --git a/microblog.py b/src/microblog.py similarity index 87% rename from microblog.py rename to src/microblog.py index bb1ba53..4e2584a 100644 --- a/microblog.py +++ b/src/microblog.py @@ -3,17 +3,15 @@ import sys, os, traceback import dateutil.parser from time import strftime, localtime -# returns html-formatted string -def make_buttons(btn_dict, msg_id): - buttons = "
" - fmt = "[%s]" - for key in btn_dict: - url = btn_dict[key] +def make_buttons(btn_conf, msg_id): + fmt = btn_conf["format"] + buttons = str() + for key in btn_conf["links"]: + url = btn_conf["links"][key] if url[-1] == '=': - # then interpret it as a query string url += str(msg_id) - buttons += fmt % (url,key) - buttons += "
" + buttons += fmt.format( + __url__=url, __label__ = key) return buttons # apply div classes for use with .css @@ -132,8 +130,6 @@ def markup(message, config): ignore = config["inline_tags"] parser = My_Html_Parser(ignore) sep = "" - if "line_separator" in config: - sep = config["line_separator"] for line in message: images = [] # list of integers parser.feed(line) @@ -172,9 +168,11 @@ class Post: return int(t.timestamp()) # format used for display - def get_short_time(self): + def get_short_time(self, form): + if form == "": + form = "%y %b %d" t = dateutil.parser.parse(self.timestamp) - return t.strftime("%y %b %d") + return t.strftime(form) def parse_txt(filename): content = [] @@ -203,7 +201,7 @@ def parse_txt(filename): state = 0 return posts -def get_posts(posts, config): +def get_posts(posts, config, newest = None): taginfos = [] tagcloud = dict() # (tag, count) tagged = dict() # (tag, index of message) @@ -211,21 +209,29 @@ def get_posts(posts, config): count = total index = count # - 1 timeline = [] - btns = None + df = "" + subset = [] + if "date_format" in config: + df = config["date_format"] for post in posts: markedup, tags = markup(post.message, config) count -= 1 index -= 1 timeline.append( - make_post(count, post.get_short_time(), config, markedup) + make_post(count, post.get_short_time(df), config, markedup) ) for tag in tags: if tagcloud.get(tag) == None: tagcloud[tag] = 0 tagcloud[tag] += 1 - if tagged.get(tag) == None: - tagged[tag] = [] - tagged[tag].append(index) + if newest is not None and (total - (1 + count)) < newest: + subset.append(tag) + if newest is None \ + or newest is not None and tag in subset: + if tagged.get(tag) == None: + tagged[tag] = [] + tagged[tag].append(index) + # print(tagged, file=sys.stderr) return timeline, tagcloud, tagged def make_tagcloud(d, rell): @@ -282,16 +288,14 @@ class Paginator: def paginate(self, template, tagcloud, timeline, is_tagline=False): outfile = "%s/%s" % (self.SUBDIR, self.FILENAME) - timeline.reverse() # reorder from oldest to newest - start = 0 - for i in range(start, self.TOTAL_PAGES): + l = len(timeline) + for i in range(0, self.TOTAL_PAGES): fn = outfile % i with open(fn, 'w') as f: self.written.append(fn) - prev = self.PPP * i - curr = self.PPP * (i+1) - sliced = timeline[prev:curr] - sliced.reverse() + prev = l - (self.PPP * i) + curr = l - self.PPP * (i+1) + sliced = timeline[curr:prev] f.write(self.singlepage(template, tagcloud, sliced, i, ".")) return @@ -315,19 +319,20 @@ if __name__ == "__main__": p = argparse.ArgumentParser() p.add_argument("template", help="an html template file") p.add_argument("content", help="text file for microblog content") - p.add_argument("--sort", \ + p.add_argument("--sort", action="store_true", \ help="sorts content from oldest to newest" - " (this is a separate operation from page generation)", \ - action="store_true") - p.add_argument("--skip-fetch", \ + " (this is a separate operation from page generation)") + p.add_argument("--skip-fetch", action="store_true", \ help="skips fetching profile data from remote sources;" - " has no effect if webring is not enabled",\ - action="store_true") + " has no effect if webring is not enabled") + p.add_argument("--new-posts", type=int, nargs='?', + help="generate pages based only on new entries; " \ + "if I wrote 5 new posts then --new-posts=5'") args = p.parse_args() if args.sort: sort(args.content) exit() - return args.template, args.content, args.skip_fetch + return args.template, args.content, args.skip_fetch, args.new_posts # assume relative path def demote_css(template, css_list, level=1): @@ -342,7 +347,7 @@ if __name__ == "__main__": tpl = tpl.replace(css, ("%s%s" % (prepend, css) )) return tpl - def writepage(template, timeline, tagcloud, config, subdir = None): + def writepage(template, timeline, tagcloud, config, subdir = None, paginate = True): count = len(timeline) html = "" with open(template,'r') as f: @@ -356,10 +361,11 @@ if __name__ == "__main__": except Exception as e: print("error: ",e, ("(number of posts = %i)" % count), file=sys.stderr) exit() + index = config["landing_page"] latest = timeline[:pagectrl.PPP] - link_from_top = "./tags/%s/latest.html" - link_from_subdir = "../tags/%s/latest.html" - link_from_tagdir = "../%s/latest.html" + link_from_top = "./tags/%s/" + index + link_from_subdir = "../tags/%s/" + index + link_from_tagdir = "../%s/" + index cloud = "" level = 1 is_tagline = False @@ -375,20 +381,20 @@ if __name__ == "__main__": else: cloud = make_tagcloud(tagcloud, link_from_subdir) demoted = demote_css(html, config["relative_css"], level) - filename = "%s/latest.html" % subdir + filename = "%s/%s" % (subdir, index) with open(filename, 'w') as f: # landing page for tag pagectrl.written.append(filename) page = pagectrl.singlepage(demoted, cloud, latest, p=".") f.write(page) - pagectrl.paginate( - demote_css(html, config["relative_css"], level), - cloud, timeline, is_tagline) + if paginate: + pagectrl.paginate( + demote_css(html, config["relative_css"], level), + cloud, timeline, is_tagline) return pagectrl.written import toml - def load_settings(): + def load_settings(filename = "settings.toml"): s = dict() - filename = "settings.toml" if os.path.exists(filename): with open(filename, 'r') as f: s = toml.loads(f.read()) @@ -457,7 +463,7 @@ if __name__ == "__main__": print(e) return json_objs - def render(profiles, template): + def render(profiles, template, date_format): rendered = [] SHORT_BIO_LIMIT = 150 for profile in profiles: @@ -478,7 +484,7 @@ if __name__ == "__main__": __post_count__ = post_count, __shortbio__= escape(self_desc), __lastupdated__= strftime( - "%Y %b %d", localtime(epoch_timestamp)) ) + date_format, localtime(epoch_timestamp)) ) rendered.append(foo) except KeyError as e: print("remote profile is missing key: ", e, file=sys.stderr) @@ -519,10 +525,9 @@ if __name__ == "__main__": try: list_of_json_objs.sort(key=lambda e: e["last-updated"], reverse=True) except KeyError: pass - return render(list_of_json_objs, f_cfg["format"]) + return render(list_of_json_objs, f_cfg["format"], f_cfg["date_format"]) - def main(): - tpl, content, skip_fetch = get_args() + def main(tpl, content, skip_fetch, new_posts): cfg = load_settings() if cfg == None: print("exit: no settings.toml found.", file=sys.stderr) @@ -534,25 +539,31 @@ if __name__ == "__main__": print("exit: table 'page' absent in settings.toml", file=sys.stderr) return p = parse_txt(content) - tl, tc, tg = get_posts(p, cfg["post"]) + tl, tc, tg = get_posts(p, cfg["post"], new_posts) if tl == []: return # main timeline updated = [] - updated += writepage(tpl, tl, tc, cfg["page"]) + updated += writepage(tpl, tl, tc, cfg["page"], + paginate=True if new_posts is None else False) # timeline per tag if tc != dict() and tg != dict(): if not os.path.exists("tags"): os.mkdir("tags") + tl.reverse() for key in tg.keys(): tagline = [] for index in tg[key]: tagline.append(tl[index]) # [1:] means to omit hashtag from dir name + wp = True # will paginate + if new_posts is not None \ + and len(tagline) > cfg["page"]["postsperpage"]: + wp = False updated += writepage( tpl, tagline, tc, cfg["page"], \ - subdir="tags/%s" % key[1:] \ - ) + subdir="tags/%s" % key[1:], \ + paginate=wp) if "webring" in cfg: if cfg["webring"]["enabled"] == True: export_profile( @@ -565,13 +576,11 @@ if __name__ == "__main__": with open("updatedfiles.txt", 'w') as f: for filename in updated: print(filename, file=f) # sys.stderr) - if "latestpage" in cfg: - print(cfg["latestpage"], file=f) if "latestpages" in cfg: for page in cfg["latestpages"]: print(page, file=f) try: - main() + main(*get_args()) except KeyError as e: traceback.print_exc() print("\n\tA key may be missing from your settings file.", file=sys.stderr) diff --git a/neouploader.py b/src/neouploader.py similarity index 100% rename from neouploader.py rename to src/neouploader.py