diff --git a/microblog.py b/microblog.py index 21a3794..12ac17e 100644 --- a/microblog.py +++ b/microblog.py @@ -365,7 +365,7 @@ if __name__ == "__main__": with open(config["file_output"], 'w') as f: print(json.dumps(p), file=f) - def fn1(webring_config): # come up with better name later/ + def fn1(f_cfg): # come up with better name later/ import pycurl from io import BytesIO def get_proxy(): @@ -388,6 +388,7 @@ if __name__ == "__main__": if is_socks: curl.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS5_HOSTNAME) datum = [] + meta = [] for url in url_list: buf = BytesIO() curl.setopt(curl.WRITEDATA, buf) @@ -395,11 +396,13 @@ if __name__ == "__main__": try: curl.perform() datum.append(buf) + meta.append(curl.getinfo(curl.CONTENT_TYPE)) except pycurl.error as e: print(e,": ", url, file=sys.stderr) # print(buf.getvalue(),"\n\t", curl.getinfo(curl.CONTENT_TYPE), file=sys.stderr) curl.close() - return datum + assert(len(datum) == len(meta)) + return datum, meta def to_json(curl_outs): json_objs = [] @@ -410,38 +413,65 @@ if __name__ == "__main__": print(e) return json_objs - list_of_json_objs = to_json(fetch(webring_config["list"])) + def render(profiles, template): + rendered = [] + SHORT_BIO_LIMIT = 150 + for profile in profiles: + try: + epoch_timestamp = profile["last-updated"] + if not isinstance(epoch_timestamp, int): + epoch_timestamp = 0 + post_count = profile["post-count"] + if not isinstance(post_count, int): + post_count = 0 + self_desc = profile["short-bio"] + if len(profile["short-bio"]) >= SHORT_BIO_LIMIT: + self_desc = profile["short-bio"][:SHORT_BIO_LIMIT] + "..." + foo = template.format( + __avatar__=escape(profile["avatar"]), + __handle__=escape(profile["username"]), + __url__=escape(profile["url"]), + __post_count__ = post_count, + __shortbio__= escape(self_desc), + __lastupdated__= strftime( + "%Y %b %d", localtime(epoch_timestamp)) ) + rendered.append(foo) + except KeyError as e: + print("remote profile is missing key: ", e, file=sys.stderr) + print("\tsource: ", profile, file=sys.stderr) + return rendered + + def fn1a(profiles, save_path, img_src): + import hashlib + avatar_urls = [] + for profile in profiles: + avatar_urls.append(profile["avatar"]) + imgs, info = fetch(avatar_urls) + l = len(imgs) + if l != len(profiles) or l == 0: + print("error in retrieving images", file=sys.stderr) + return + for i in range(0, l): + image = imgs[i] + ext = info[i].split('/').pop() + data = image.getvalue() + h = hashlib.sha1(data).hexdigest() + filename = "%s.%s" % (h, ext) + with open("%s/%s" % (save_path, filename), "wb") as f: + f.write(data) + profiles[i]["avatar"] = "%s/%s" % (img_src, filename) + + j, m = fetch(f_cfg["list"]) + list_of_json_objs = to_json(j) if list_of_json_objs == []: print("no remote profiles loaded", file=sys.stderr) return [] + if f_cfg["internal-avatars"]["enabled"]: + a = f_cfg["internal-avatars"]["local_path_to_avatars"] + b = f_cfg["internal-avatars"]["path_to_avatars"] + fn1a(list_of_json_objs, a, b) list_of_json_objs.sort(key=lambda e: e["last-updated"], reverse=True) - rendered = [] - template = webring_config["format"] - SHORT_BIO_LIMIT = 150 - for profile in list_of_json_objs: - try: - epoch_timestamp = profile["last-updated"] - if not isinstance(epoch_timestamp, int): - epoch_timestamp = 0 - post_count = profile["post-count"] - if not isinstance(post_count, int): - post_count = 0 - self_desc = profile["short-bio"] - if len(profile["short-bio"]) >= SHORT_BIO_LIMIT: - self_desc = profile["short-bio"][:SHORT_BIO_LIMIT] + "..." - foo = template.format( - __avatar__=escape(profile["avatar"]), - __handle__=escape(profile["username"]), - __url__=escape(profile["url"]), - __post_count__ = post_count, - __shortbio__= escape(self_desc), - __lastupdated__= strftime( - "%Y %b %d", localtime(epoch_timestamp)) ) - except KeyError as e: - print("remote profile is missing key: ", e, file=sys.stderr) - print("\tsource: ", profile, file=sys.stderr) - rendered.append(foo) - return rendered + return render(list_of_json_objs, f_cfg["format"]) def main(): tpl, content = get_args() @@ -479,11 +509,10 @@ if __name__ == "__main__": if cfg["webring"]["enabled"] == True: export_profile( len(p), p[0].get_epoch_time(), cfg["webring"] ) - if "following" in cfg["webring"]: - fellows = fn1(cfg["webring"]["following"] ) - if fellows != []: - updated += writepage( - tpl, fellows, tc, cfg["page"], subdir="placeholder" ) + fellows = fn1(cfg["webring"]["following"] ) + if fellows != []: + updated += writepage( + tpl, fellows, tc, cfg["page"], subdir="placeholder" ) with open("updatedfiles.txt", 'w') as f: for filename in updated: print(filename, file=f) # sys.stderr) @@ -505,4 +534,6 @@ if __name__ == "__main__": except toml.decoder.TomlDecodeError: traceback.print_exc() print("\n\tYour configuration file is malformed.") - + except FileNotFoundError as e: + traceback.print_exc() + print("\n\tA potential cause is attempting to save a file to a folder that does not exist..")