using pycurl; dont need urllib.request or pysocks

microblog.py

@@ -373,45 +373,47 @@ if __name__ == "__main__":
         with open(config["file_output"], 'w') as f:
             print(json.dumps(profile), file=f)
 
-    def fn1(webring_config):    # come up with better name later
+    def fn1(webring_config):    # come up with better name later/
-        def fetch(follow_list):
+        def get_proxy():
-            import urllib.request as http
+            proxy = ""
-            def support_tor(proxy_var="https_proxy"):
+            if "http_proxy" in os.environ:
-                if proxy_var not in os.environ:
+                proxy = os.environ["http_proxy"]
-                    return False
+            elif "https_proxy" in os.environ:
-                proxy = os.environ[proxy_var]
+                proxy = os.environ["https_proxy"]
-                if proxy.find("socks://") < 0:
-                    return False
-                import socket
-                import socks
             host = proxy[proxy.rfind('/') + 1: proxy.rfind(':')]
             port = proxy[proxy.rfind(':') + 1:]
-                socks.setdefaultproxy(
+            foo = proxy.find("socks://") >= 0 or proxy.find("socks5h://")
-                    socks.PROXY_TYPE_SOCKS5, host, int(port))
+            return host, int(port), foo
-                socket.socket = socks.socksocket
-                return True
 
+        def fetch(follow_list):
+            import pycurl
+            from io import BytesIO
+            curl = pycurl.Curl() 
+            if "http_proxy" in os.environ or "https_proxy" in os.environ:
+                hostname, port_no, is_socks = get_proxy()
+                curl.setopt(pycurl.PROXY, hostname)
+                curl.setopt(pycurl.PROXYPORT, port_no)
+                if is_socks:
+                    curl.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS5_HOSTNAME)
             other_people = []
-            proxy_handler = http.ProxyHandler(None) # use OS vars
-            if support_tor("https_proxy") or support_tor("http_proxy"):
-                proxy_handler = http.ProxyHandler({})
-            conn = http.build_opener(proxy_handler)
             for someone in follow_list:
+                buf = BytesIO()
+                curl.setopt(curl.WRITEDATA, buf)
+                curl.setopt(pycurl.URL, someone)
                 try:
-                    with conn.open(someone) as response:
+                    curl.perform()
-                        data = response.read()
+                    other_people.append(json.loads(buf.getvalue()))
-                        other_people.append(json.loads(data))
+                except pycurl.error as e:
-                except http.HTTPError as e:
                     print(e,": ", someone, file=sys.stderr)
-                    pass
+                # print(buf.getvalue(),"\n\t", curl.getinfo(curl.CONTENT_TYPE), file=sys.stderr)
-                except http.URLError as e: 
+            curl.close()
-                    print(e, file=sys.stderr)
-                    print("\t =>", someone, file=sys.stderr)
             return other_people
 
         list_of_json_objs = fetch(webring_config["list"])
-        list_of_json_objs.sort(
+        try:
-            key=lambda e: e["last-updated"], reverse=True)
+            list_of_json_objs.sort(key=lambda e: e["last-updated"], reverse=True)
+        except:
+            pass
         if list_of_json_objs == []:
             print("no remote profiles loaded", file=sys.stderr)
             return []
@@ -419,27 +421,28 @@ if __name__ == "__main__":
         template = webring_config["format"]
         SHORT_BIO_LIMIT = 150
         for profile in list_of_json_objs:
+            try:
                 epoch_timestamp = profile["last-updated"]
                 if not isinstance(epoch_timestamp, int):
                     epoch_timestamp = 0
-
                 post_count = profile["post-count"]
                 if not isinstance(post_count, int):
                     post_count = 0
-
                 self_desc = profile["short-bio"]
                 if len(profile["short-bio"]) >= SHORT_BIO_LIMIT:
                     self_desc = profile["short-bio"][:SHORT_BIO_LIMIT] + "..."
-
+                foo = template.format(
-            rendered.append(template.format(
                     __avatar__=escape(profile["avatar"]),
                     __handle__=escape(profile["username"]),
                     __url__=escape(profile["url"]),
                     __post_count__ = post_count,
                     __shortbio__= escape(self_desc),
                     __lastupdated__= strftime(
-                    "%Y %b %d", localtime(epoch_timestamp))
+                        "%Y %b %d", localtime(epoch_timestamp)) )
-            ))
+            except KeyError as e:
+                print("remote profile is missing key: ", e, file=sys.stderr)
+                print("\tsource: ", profile, file=sys.stderr)
+            rendered.append(foo)
         return rendered
 
     def main():
@@ -504,6 +507,4 @@ if __name__ == "__main__":
     except toml.decoder.TomlDecodeError:
         traceback.print_exc()
         print("\n\tYour configuration file is malformed.")
-    except ModuleNotFoundError:
+
-        traceback.print_exc()
-        print("\n\tIf the module is 'socks', Tor support has recently been added (install PySocks package)")