likho/microblog.py
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

using pycurl; dont need urllib.request or pysocks

Browse Source
This commit is contained in:
likho 2023-08-09 02:00:43 -07:00
parent e4af5bbd6a
commit 3bf37386fc
1 changed files with 56 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

111
microblog.py
View File

@ -373,45 +373,47 @@ if __name__ == "__main__":
with open(config["file_output"], 'w') as f: with open(config["file_output"], 'w') as f:
print(json.dumps(profile), file=f) print(json.dumps(profile), file=f)
def fn1(webring_config): # come up with better name later def fn1(webring_config): # come up with better name later/
def fetch(follow_list): def get_proxy():
import urllib.request as http proxy = ""
def support_tor(proxy_var="https_proxy"): if "http_proxy" in os.environ:
if proxy_var not in os.environ: proxy = os.environ["http_proxy"]
return False elif "https_proxy" in os.environ:
proxy = os.environ[proxy_var] proxy = os.environ["https_proxy"]
if proxy.find("socks://") < 0: host = proxy[proxy.rfind('/') + 1: proxy.rfind(':')]
return False port = proxy[proxy.rfind(':') + 1:]
import socket foo = proxy.find("socks://") >= 0 or proxy.find("socks5h://")
import socks return host, int(port), foo
host = proxy[proxy.rfind('/') + 1: proxy.rfind(':')]
port = proxy[proxy.rfind(':') + 1:]
socks.setdefaultproxy(
socks.PROXY_TYPE_SOCKS5, host, int(port))
socket.socket = socks.socksocket
return True
def fetch(follow_list):
import pycurl
from io import BytesIO
curl = pycurl.Curl()
if "http_proxy" in os.environ or "https_proxy" in os.environ:
hostname, port_no, is_socks = get_proxy()
curl.setopt(pycurl.PROXY, hostname)
curl.setopt(pycurl.PROXYPORT, port_no)
if is_socks:
curl.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS5_HOSTNAME)
other_people = [] other_people = []
proxy_handler = http.ProxyHandler(None) # use OS vars
if support_tor("https_proxy") or support_tor("http_proxy"):
proxy_handler = http.ProxyHandler({})
conn = http.build_opener(proxy_handler)
for someone in follow_list: for someone in follow_list:
buf = BytesIO()
curl.setopt(curl.WRITEDATA, buf)
curl.setopt(pycurl.URL, someone)
try: try:
with conn.open(someone) as response: curl.perform()
data = response.read() other_people.append(json.loads(buf.getvalue()))
other_people.append(json.loads(data)) except pycurl.error as e:
except http.HTTPError as e:
print(e,": ", someone, file=sys.stderr) print(e,": ", someone, file=sys.stderr)
pass # print(buf.getvalue(),"\n\t", curl.getinfo(curl.CONTENT_TYPE), file=sys.stderr)
except http.URLError as e: curl.close()
print(e, file=sys.stderr)
print("\t =>", someone, file=sys.stderr)
return other_people return other_people
list_of_json_objs = fetch(webring_config["list"]) list_of_json_objs = fetch(webring_config["list"])
list_of_json_objs.sort( try:
key=lambda e: e["last-updated"], reverse=True) list_of_json_objs.sort(key=lambda e: e["last-updated"], reverse=True)
except:
pass
if list_of_json_objs == []: if list_of_json_objs == []:
print("no remote profiles loaded", file=sys.stderr) print("no remote profiles loaded", file=sys.stderr)
return [] return []
@ -419,27 +421,28 @@ if __name__ == "__main__":
template = webring_config["format"] template = webring_config["format"]
SHORT_BIO_LIMIT = 150 SHORT_BIO_LIMIT = 150
for profile in list_of_json_objs: for profile in list_of_json_objs:
epoch_timestamp = profile["last-updated"] try:
if not isinstance(epoch_timestamp, int): epoch_timestamp = profile["last-updated"]
epoch_timestamp = 0 if not isinstance(epoch_timestamp, int):
epoch_timestamp = 0
post_count = profile["post-count"] post_count = profile["post-count"]
if not isinstance(post_count, int): if not isinstance(post_count, int):
post_count = 0 post_count = 0
self_desc = profile["short-bio"]
self_desc = profile["short-bio"] if len(profile["short-bio"]) >= SHORT_BIO_LIMIT:
if len(profile["short-bio"]) >= SHORT_BIO_LIMIT: self_desc = profile["short-bio"][:SHORT_BIO_LIMIT] + "..."
self_desc = profile["short-bio"][:SHORT_BIO_LIMIT] + "..." foo = template.format(
__avatar__=escape(profile["avatar"]),
rendered.append(template.format( __handle__=escape(profile["username"]),
__avatar__=escape(profile["avatar"]), __url__=escape(profile["url"]),
__handle__=escape(profile["username"]), __post_count__ = post_count,
__url__=escape(profile["url"]), __shortbio__= escape(self_desc),
__post_count__ = post_count, __lastupdated__= strftime(
__shortbio__= escape(self_desc), "%Y %b %d", localtime(epoch_timestamp)) )
__lastupdated__= strftime( except KeyError as e:
"%Y %b %d", localtime(epoch_timestamp)) print("remote profile is missing key: ", e, file=sys.stderr)
)) print("\tsource: ", profile, file=sys.stderr)
rendered.append(foo)
return rendered return rendered
def main(): def main():
@ -504,6 +507,4 @@ if __name__ == "__main__":
except toml.decoder.TomlDecodeError: except toml.decoder.TomlDecodeError:
traceback.print_exc() traceback.print_exc()
print("\n\tYour configuration file is malformed.") print("\n\tYour configuration file is malformed.")
except ModuleNotFoundError:
traceback.print_exc()
print("\n\tIf the module is 'socks', Tor support has recently been added (install PySocks package)")