generalized output of fetch

microblog.py

@@ -374,6 +374,8 @@ if __name__ == "__main__":
             print(json.dumps(profile), file=f)
 
     def fn1(webring_config):    # come up with better name later/
+        import pycurl
+        from io import BytesIO
         def get_proxy():
             proxy = ""
             if "http_proxy" in os.environ:
@@ -385,9 +387,7 @@ if __name__ == "__main__":
             foo = proxy.find("socks://") >= 0 or proxy.find("socks5h://")
             return host, int(port), foo
 
-        def fetch(follow_list):
-            import pycurl
-            from io import BytesIO
+        def fetch(url_list):
             curl = pycurl.Curl() 
             if "http_proxy" in os.environ or "https_proxy" in os.environ:
                 hostname, port_no, is_socks = get_proxy()
@@ -395,28 +395,34 @@ if __name__ == "__main__":
                 curl.setopt(pycurl.PROXYPORT, port_no)
                 if is_socks:
                     curl.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS5_HOSTNAME)
-            other_people = []
-            for someone in follow_list:
+            datum = []
+            for url in url_list:
                 buf = BytesIO()
                 curl.setopt(curl.WRITEDATA, buf)
-                curl.setopt(pycurl.URL, someone)
+                curl.setopt(pycurl.URL, url)
                 try:
                     curl.perform()
-                    other_people.append(json.loads(buf.getvalue()))
+                    datum.append(buf)
                 except pycurl.error as e:
-                    print(e,": ", someone, file=sys.stderr)
+                    print(e,": ", url, file=sys.stderr)
                 # print(buf.getvalue(),"\n\t", curl.getinfo(curl.CONTENT_TYPE), file=sys.stderr)
             curl.close()
-            return other_people
+            return datum
 
-        list_of_json_objs = fetch(webring_config["list"])
-        try:
-            list_of_json_objs.sort(key=lambda e: e["last-updated"], reverse=True)
-        except:
-            pass
+        def to_json(curl_outs):
+            json_objs = []
+            for buf in curl_outs:
+                try:
+                    json_objs.append(json.loads(buf.getvalue()))
+                except Exception as e:
+                    print(e)
+            return json_objs
+
+        list_of_json_objs = to_json(fetch(webring_config["list"]))
         if list_of_json_objs == []:
             print("no remote profiles loaded", file=sys.stderr)
             return []
+        list_of_json_objs.sort(key=lambda e: e["last-updated"], reverse=True)
         rendered = []
         template = webring_config["format"]
         SHORT_BIO_LIMIT = 150