added functionality to avoid hotlinking to avatars

microblog.py

@@ -365,7 +365,7 @@ if __name__ == "__main__":
         with open(config["file_output"], 'w') as f:
             print(json.dumps(p), file=f)
 
-    def fn1(webring_config):    # come up with better name later/
+    def fn1(f_cfg):    # come up with better name later/
         import pycurl
         from io import BytesIO
         def get_proxy():
@@ -388,6 +388,7 @@ if __name__ == "__main__":
                 if is_socks:
                     curl.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS5_HOSTNAME)
             datum = []
+            meta  = []
             for url in url_list:
                 buf = BytesIO()
                 curl.setopt(curl.WRITEDATA, buf)
@@ -395,11 +396,13 @@ if __name__ == "__main__":
                 try:
                     curl.perform()
                     datum.append(buf)
+                    meta.append(curl.getinfo(curl.CONTENT_TYPE))
                 except pycurl.error as e:
                     print(e,": ", url, file=sys.stderr)
                 # print(buf.getvalue(),"\n\t", curl.getinfo(curl.CONTENT_TYPE), file=sys.stderr)
             curl.close()
-            return datum
+            assert(len(datum) == len(meta))
+            return datum, meta
 
         def to_json(curl_outs):
             json_objs = []
@@ -410,38 +413,65 @@ if __name__ == "__main__":
                     print(e)
             return json_objs
 
-        list_of_json_objs = to_json(fetch(webring_config["list"]))
+        def render(profiles, template):
+            rendered = []
+            SHORT_BIO_LIMIT = 150
+            for profile in profiles:
+                try:
+                    epoch_timestamp = profile["last-updated"]
+                    if not isinstance(epoch_timestamp, int):
+                        epoch_timestamp = 0
+                    post_count = profile["post-count"]
+                    if not isinstance(post_count, int):
+                        post_count = 0
+                    self_desc = profile["short-bio"]
+                    if len(profile["short-bio"]) >= SHORT_BIO_LIMIT:
+                        self_desc = profile["short-bio"][:SHORT_BIO_LIMIT] + "..."
+                    foo = template.format(
+                        __avatar__=escape(profile["avatar"]),
+                        __handle__=escape(profile["username"]),
+                        __url__=escape(profile["url"]),
+                        __post_count__ = post_count,
+                        __shortbio__= escape(self_desc),
+                        __lastupdated__= strftime(
+                            "%Y %b %d", localtime(epoch_timestamp)) )
+                    rendered.append(foo)
+                except KeyError as e:
+                    print("remote profile is missing key: ", e, file=sys.stderr)
+                    print("\tsource: ", profile, file=sys.stderr)
+            return rendered
+
+        def fn1a(profiles, save_path, img_src):
+            import hashlib
+            avatar_urls = []
+            for profile in profiles:
+                avatar_urls.append(profile["avatar"])
+            imgs, info = fetch(avatar_urls)
+            l = len(imgs)
+            if l != len(profiles) or l == 0:
+                print("error in retrieving images", file=sys.stderr)
+                return
+            for i in range(0, l):
+                image = imgs[i]
+                ext   = info[i].split('/').pop()
+                data  = image.getvalue()
+                h = hashlib.sha1(data).hexdigest()
+                filename = "%s.%s" % (h, ext)
+                with open("%s/%s" % (save_path, filename), "wb") as f:
+                    f.write(data)
+                profiles[i]["avatar"] = "%s/%s" % (img_src, filename)
+
+        j, m = fetch(f_cfg["list"])
+        list_of_json_objs = to_json(j)
         if list_of_json_objs == []:
             print("no remote profiles loaded", file=sys.stderr)
             return []
+        if f_cfg["internal-avatars"]["enabled"]:
+            a = f_cfg["internal-avatars"]["local_path_to_avatars"]
+            b = f_cfg["internal-avatars"]["path_to_avatars"]
+            fn1a(list_of_json_objs, a, b)
         list_of_json_objs.sort(key=lambda e: e["last-updated"], reverse=True)
-        rendered = []
+        return render(list_of_json_objs, f_cfg["format"])
-        template = webring_config["format"]
-        SHORT_BIO_LIMIT = 150
-        for profile in list_of_json_objs:
-            try:
-                epoch_timestamp = profile["last-updated"]
-                if not isinstance(epoch_timestamp, int):
-                    epoch_timestamp = 0
-                post_count = profile["post-count"]
-                if not isinstance(post_count, int):
-                    post_count = 0
-                self_desc = profile["short-bio"]
-                if len(profile["short-bio"]) >= SHORT_BIO_LIMIT:
-                    self_desc = profile["short-bio"][:SHORT_BIO_LIMIT] + "..."
-                foo = template.format(
-                    __avatar__=escape(profile["avatar"]),
-                    __handle__=escape(profile["username"]),
-                    __url__=escape(profile["url"]),
-                    __post_count__ = post_count,
-                    __shortbio__= escape(self_desc),
-                    __lastupdated__= strftime(
-                        "%Y %b %d", localtime(epoch_timestamp)) )
-            except KeyError as e:
-                print("remote profile is missing key: ", e, file=sys.stderr)
-                print("\tsource: ", profile, file=sys.stderr)
-            rendered.append(foo)
-        return rendered
 
     def main():
         tpl, content = get_args()
@@ -479,11 +509,10 @@ if __name__ == "__main__":
             if cfg["webring"]["enabled"] == True:
                 export_profile(
                     len(p), p[0].get_epoch_time(), cfg["webring"] )
-                if "following" in cfg["webring"]:
+                fellows = fn1(cfg["webring"]["following"] )
-                    fellows = fn1(cfg["webring"]["following"] )
+                if fellows != []:
-                    if fellows != []:
+                    updated += writepage(
-                        updated += writepage(
+                        tpl, fellows, tc, cfg["page"], subdir="placeholder" )
-                            tpl, fellows, tc, cfg["page"], subdir="placeholder" )
         with open("updatedfiles.txt", 'w') as f:
             for filename in updated:
                 print(filename, file=f) # sys.stderr)
@@ -505,4 +534,6 @@ if __name__ == "__main__":
     except toml.decoder.TomlDecodeError:
         traceback.print_exc()
         print("\n\tYour configuration file is malformed.")
-
+    except FileNotFoundError as e:
+        traceback.print_exc()
+        print("\n\tA potential cause is attempting to save a file to a folder that does not exist..")