diff --git a/microblog.py b/microblog.py index 6772adc..cb9f05b 100644 --- a/microblog.py +++ b/microblog.py @@ -373,45 +373,47 @@ if __name__ == "__main__": with open(config["file_output"], 'w') as f: print(json.dumps(profile), file=f) - def fn1(webring_config): # come up with better name later - def fetch(follow_list): - import urllib.request as http - def support_tor(proxy_var="https_proxy"): - if proxy_var not in os.environ: - return False - proxy = os.environ[proxy_var] - if proxy.find("socks://") < 0: - return False - import socket - import socks - host = proxy[proxy.rfind('/') + 1: proxy.rfind(':')] - port = proxy[proxy.rfind(':') + 1:] - socks.setdefaultproxy( - socks.PROXY_TYPE_SOCKS5, host, int(port)) - socket.socket = socks.socksocket - return True + def fn1(webring_config): # come up with better name later/ + def get_proxy(): + proxy = "" + if "http_proxy" in os.environ: + proxy = os.environ["http_proxy"] + elif "https_proxy" in os.environ: + proxy = os.environ["https_proxy"] + host = proxy[proxy.rfind('/') + 1: proxy.rfind(':')] + port = proxy[proxy.rfind(':') + 1:] + foo = proxy.find("socks://") >= 0 or proxy.find("socks5h://") + return host, int(port), foo + def fetch(follow_list): + import pycurl + from io import BytesIO + curl = pycurl.Curl() + if "http_proxy" in os.environ or "https_proxy" in os.environ: + hostname, port_no, is_socks = get_proxy() + curl.setopt(pycurl.PROXY, hostname) + curl.setopt(pycurl.PROXYPORT, port_no) + if is_socks: + curl.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS5_HOSTNAME) other_people = [] - proxy_handler = http.ProxyHandler(None) # use OS vars - if support_tor("https_proxy") or support_tor("http_proxy"): - proxy_handler = http.ProxyHandler({}) - conn = http.build_opener(proxy_handler) for someone in follow_list: + buf = BytesIO() + curl.setopt(curl.WRITEDATA, buf) + curl.setopt(pycurl.URL, someone) try: - with conn.open(someone) as response: - data = response.read() - other_people.append(json.loads(data)) - except http.HTTPError as e: + curl.perform() + other_people.append(json.loads(buf.getvalue())) + except pycurl.error as e: print(e,": ", someone, file=sys.stderr) - pass - except http.URLError as e: - print(e, file=sys.stderr) - print("\t =>", someone, file=sys.stderr) + # print(buf.getvalue(),"\n\t", curl.getinfo(curl.CONTENT_TYPE), file=sys.stderr) + curl.close() return other_people list_of_json_objs = fetch(webring_config["list"]) - list_of_json_objs.sort( - key=lambda e: e["last-updated"], reverse=True) + try: + list_of_json_objs.sort(key=lambda e: e["last-updated"], reverse=True) + except: + pass if list_of_json_objs == []: print("no remote profiles loaded", file=sys.stderr) return [] @@ -419,27 +421,28 @@ if __name__ == "__main__": template = webring_config["format"] SHORT_BIO_LIMIT = 150 for profile in list_of_json_objs: - epoch_timestamp = profile["last-updated"] - if not isinstance(epoch_timestamp, int): - epoch_timestamp = 0 - - post_count = profile["post-count"] - if not isinstance(post_count, int): - post_count = 0 - - self_desc = profile["short-bio"] - if len(profile["short-bio"]) >= SHORT_BIO_LIMIT: - self_desc = profile["short-bio"][:SHORT_BIO_LIMIT] + "..." - - rendered.append(template.format( - __avatar__=escape(profile["avatar"]), - __handle__=escape(profile["username"]), - __url__=escape(profile["url"]), - __post_count__ = post_count, - __shortbio__= escape(self_desc), - __lastupdated__= strftime( - "%Y %b %d", localtime(epoch_timestamp)) - )) + try: + epoch_timestamp = profile["last-updated"] + if not isinstance(epoch_timestamp, int): + epoch_timestamp = 0 + post_count = profile["post-count"] + if not isinstance(post_count, int): + post_count = 0 + self_desc = profile["short-bio"] + if len(profile["short-bio"]) >= SHORT_BIO_LIMIT: + self_desc = profile["short-bio"][:SHORT_BIO_LIMIT] + "..." + foo = template.format( + __avatar__=escape(profile["avatar"]), + __handle__=escape(profile["username"]), + __url__=escape(profile["url"]), + __post_count__ = post_count, + __shortbio__= escape(self_desc), + __lastupdated__= strftime( + "%Y %b %d", localtime(epoch_timestamp)) ) + except KeyError as e: + print("remote profile is missing key: ", e, file=sys.stderr) + print("\tsource: ", profile, file=sys.stderr) + rendered.append(foo) return rendered def main(): @@ -504,6 +507,4 @@ if __name__ == "__main__": except toml.decoder.TomlDecodeError: traceback.print_exc() print("\n\tYour configuration file is malformed.") - except ModuleNotFoundError: - traceback.print_exc() - print("\n\tIf the module is 'socks', Tor support has recently been added (install PySocks package)") +