using pycurl; dont need urllib.request or pysocks
This commit is contained in:
parent
e4af5bbd6a
commit
3bf37386fc
111
microblog.py
111
microblog.py
@ -373,45 +373,47 @@ if __name__ == "__main__":
|
||||
with open(config["file_output"], 'w') as f:
|
||||
print(json.dumps(profile), file=f)
|
||||
|
||||
def fn1(webring_config): # come up with better name later
|
||||
def fetch(follow_list):
|
||||
import urllib.request as http
|
||||
def support_tor(proxy_var="https_proxy"):
|
||||
if proxy_var not in os.environ:
|
||||
return False
|
||||
proxy = os.environ[proxy_var]
|
||||
if proxy.find("socks://") < 0:
|
||||
return False
|
||||
import socket
|
||||
import socks
|
||||
host = proxy[proxy.rfind('/') + 1: proxy.rfind(':')]
|
||||
port = proxy[proxy.rfind(':') + 1:]
|
||||
socks.setdefaultproxy(
|
||||
socks.PROXY_TYPE_SOCKS5, host, int(port))
|
||||
socket.socket = socks.socksocket
|
||||
return True
|
||||
def fn1(webring_config): # come up with better name later/
|
||||
def get_proxy():
|
||||
proxy = ""
|
||||
if "http_proxy" in os.environ:
|
||||
proxy = os.environ["http_proxy"]
|
||||
elif "https_proxy" in os.environ:
|
||||
proxy = os.environ["https_proxy"]
|
||||
host = proxy[proxy.rfind('/') + 1: proxy.rfind(':')]
|
||||
port = proxy[proxy.rfind(':') + 1:]
|
||||
foo = proxy.find("socks://") >= 0 or proxy.find("socks5h://")
|
||||
return host, int(port), foo
|
||||
|
||||
def fetch(follow_list):
|
||||
import pycurl
|
||||
from io import BytesIO
|
||||
curl = pycurl.Curl()
|
||||
if "http_proxy" in os.environ or "https_proxy" in os.environ:
|
||||
hostname, port_no, is_socks = get_proxy()
|
||||
curl.setopt(pycurl.PROXY, hostname)
|
||||
curl.setopt(pycurl.PROXYPORT, port_no)
|
||||
if is_socks:
|
||||
curl.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS5_HOSTNAME)
|
||||
other_people = []
|
||||
proxy_handler = http.ProxyHandler(None) # use OS vars
|
||||
if support_tor("https_proxy") or support_tor("http_proxy"):
|
||||
proxy_handler = http.ProxyHandler({})
|
||||
conn = http.build_opener(proxy_handler)
|
||||
for someone in follow_list:
|
||||
buf = BytesIO()
|
||||
curl.setopt(curl.WRITEDATA, buf)
|
||||
curl.setopt(pycurl.URL, someone)
|
||||
try:
|
||||
with conn.open(someone) as response:
|
||||
data = response.read()
|
||||
other_people.append(json.loads(data))
|
||||
except http.HTTPError as e:
|
||||
curl.perform()
|
||||
other_people.append(json.loads(buf.getvalue()))
|
||||
except pycurl.error as e:
|
||||
print(e,": ", someone, file=sys.stderr)
|
||||
pass
|
||||
except http.URLError as e:
|
||||
print(e, file=sys.stderr)
|
||||
print("\t =>", someone, file=sys.stderr)
|
||||
# print(buf.getvalue(),"\n\t", curl.getinfo(curl.CONTENT_TYPE), file=sys.stderr)
|
||||
curl.close()
|
||||
return other_people
|
||||
|
||||
list_of_json_objs = fetch(webring_config["list"])
|
||||
list_of_json_objs.sort(
|
||||
key=lambda e: e["last-updated"], reverse=True)
|
||||
try:
|
||||
list_of_json_objs.sort(key=lambda e: e["last-updated"], reverse=True)
|
||||
except:
|
||||
pass
|
||||
if list_of_json_objs == []:
|
||||
print("no remote profiles loaded", file=sys.stderr)
|
||||
return []
|
||||
@ -419,27 +421,28 @@ if __name__ == "__main__":
|
||||
template = webring_config["format"]
|
||||
SHORT_BIO_LIMIT = 150
|
||||
for profile in list_of_json_objs:
|
||||
epoch_timestamp = profile["last-updated"]
|
||||
if not isinstance(epoch_timestamp, int):
|
||||
epoch_timestamp = 0
|
||||
|
||||
post_count = profile["post-count"]
|
||||
if not isinstance(post_count, int):
|
||||
post_count = 0
|
||||
|
||||
self_desc = profile["short-bio"]
|
||||
if len(profile["short-bio"]) >= SHORT_BIO_LIMIT:
|
||||
self_desc = profile["short-bio"][:SHORT_BIO_LIMIT] + "..."
|
||||
|
||||
rendered.append(template.format(
|
||||
__avatar__=escape(profile["avatar"]),
|
||||
__handle__=escape(profile["username"]),
|
||||
__url__=escape(profile["url"]),
|
||||
__post_count__ = post_count,
|
||||
__shortbio__= escape(self_desc),
|
||||
__lastupdated__= strftime(
|
||||
"%Y %b %d", localtime(epoch_timestamp))
|
||||
))
|
||||
try:
|
||||
epoch_timestamp = profile["last-updated"]
|
||||
if not isinstance(epoch_timestamp, int):
|
||||
epoch_timestamp = 0
|
||||
post_count = profile["post-count"]
|
||||
if not isinstance(post_count, int):
|
||||
post_count = 0
|
||||
self_desc = profile["short-bio"]
|
||||
if len(profile["short-bio"]) >= SHORT_BIO_LIMIT:
|
||||
self_desc = profile["short-bio"][:SHORT_BIO_LIMIT] + "..."
|
||||
foo = template.format(
|
||||
__avatar__=escape(profile["avatar"]),
|
||||
__handle__=escape(profile["username"]),
|
||||
__url__=escape(profile["url"]),
|
||||
__post_count__ = post_count,
|
||||
__shortbio__= escape(self_desc),
|
||||
__lastupdated__= strftime(
|
||||
"%Y %b %d", localtime(epoch_timestamp)) )
|
||||
except KeyError as e:
|
||||
print("remote profile is missing key: ", e, file=sys.stderr)
|
||||
print("\tsource: ", profile, file=sys.stderr)
|
||||
rendered.append(foo)
|
||||
return rendered
|
||||
|
||||
def main():
|
||||
@ -504,6 +507,4 @@ if __name__ == "__main__":
|
||||
except toml.decoder.TomlDecodeError:
|
||||
traceback.print_exc()
|
||||
print("\n\tYour configuration file is malformed.")
|
||||
except ModuleNotFoundError:
|
||||
traceback.print_exc()
|
||||
print("\n\tIf the module is 'socks', Tor support has recently been added (install PySocks package)")
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user