Source code for bbarchivist.networkutils
#!/usr/bin/env python3
import os # filesystem read
import requests # downloading
import time # time for downloader
import queue # downloader multithreading
import threading # downloader multithreading
import binascii # downloader thread naming
import math # rounding of floats
import xml.etree.ElementTree # XML parsing
[docs]class Downloader(threading.Thread):
"""
Downloads files attached to supplied threads from DownloadManager.
Based on:
http://pipe-devnull.com/2012/09/13/queued-threaded-http-downloader-in-python.html
"""
def __init__(self, queue, output_directory):
"""
Initiate downloader thread.
:param queue: What to download.
:type queue: dict
:param output_directory: Where to output.
:type output_directory: str
"""
threading.Thread.__init__(self, name=binascii.hexlify(os.urandom(8)))
self.queue = queue
self.output_directory = output_directory
[docs] def run(self):
"""
Download files from queue.
"""
while True:
# gets the url from the queue
url = self.queue.get()
# download the file
self.download(url)
# send a signal to the queue that the job is done
self.queue.task_done()
[docs] def download(self, url):
"""
Download file from given URL.
:param url: URL to download from.
:type url: str
"""
t_start = time.clock()
local_filename = url.split('/')[-1]
print("Downloading:", local_filename)
r = requests.get(url, stream=True)
if (r.status_code != 404): # 200 OK
fname = self.output_directory + "/" + os.path.basename(url)
with open(fname, "wb") as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
f.write(chunk)
f.flush()
t_elapsed = time.clock() - t_start
t_elapsed_proper = math.ceil(t_elapsed * 100) / 100
print(
"Downloaded " +
local_filename +
" in " +
str(t_elapsed_proper) +
" seconds")
f.close()
else:
print("* Thread: " + self.name + " Bad URL: " + url)
return
[docs]class DownloadManager():
"""
Class that handles queued downloads.
Based on:
http://pipe-devnull.com/2012/09/13/queued-threaded-http-downloader-in-python.html
"""
def __init__(self, download_dict, output_directory, thread_count=5):
"""
Initiate download manager.
:param download_dict: Dictionary of download URLs.
:type download_dict: dict
:param output_directory: Where to output.
:type output_directory: str
:param thread_count: Number of threads. 5 by default.
:type thread_count: int
"""
self.thread_count = thread_count
self.download_dict = download_dict
self.output_directory = output_directory\
# Start the downloader threads, fill the queue with the URLs and
# then feed the threads URLs via the queue
[docs] def begin_downloads(self):
"""
Start :class:`Downloader` threads for queued downloads.
"""
dlqueue = queue.Queue()
# Create i thread pool and give them a queue
for t in range(self.thread_count):
t = Downloader(dlqueue, self.output_directory)
t.setDaemon(True)
t.start()
# Load the queue from the download dict
for linkname in self.download_dict:
# print uri
dlqueue.put(self.download_dict[linkname])
# Wait for the queue to finish
dlqueue.join()
return
[docs]def availability(url):
"""
Check HTTP status code of given URL.
200 or 301-308 is OK, else is not.
:param url: URL to check.
:type url: str
"""
try:
av = requests.head(str(url))
except requests.ConnectionError:
return False
else:
status = int(av.status_code)
if (status == 200) or (300 < status <= 308):
return True
else:
return False
[docs]def carrier_checker(mcc, mnc):
"""
Query BlackBerry World to map a MCC and a MNC to a country and carrier.
:param mcc: Country code.
:type mcc: int
:param mnc: Network code.
:type mnc: int
"""
url = "http://appworld.blackberry.com/ClientAPI/checkcarrier?homemcc="
url += str(mcc)
url += "&homemnc="
url += str(mnc)
url += "&devicevendorid=-1&pin=0"
user_agent = {'User-agent': 'AppWorld/5.1.0.60'}
r = requests.get(url, headers=user_agent)
root = xml.etree.ElementTree.fromstring(r.text)
for child in root:
if child.tag == "country":
country = child.get("name")
if child.tag == "carrier":
carrier = child.get("name")
return country, carrier
[docs]def carrier_update_request(mcc, mnc, device,
download=False, upgrade=False, blitz=False):
"""
Query BlackBerry servers, check which update is out for a carrier.
:param mcc: Country code.
:type mcc: int
:param mnc: Network code.
:type mnc: int
:param device: Hexadecimal hardware ID.
:type device: str
:param download: Whether to download files. False by default.
:type download: bool
:param upgrade: Whether to use upgrade files. False by default.
:type upgrade: bool
:param blitz: Whether or not to create a blitz package. False by default.
:type blitz: bool
"""
if upgrade:
upg = "upgrade"
else:
upg = "repair"
npc = str(mcc).zfill(3) + str(mnc).zfill(3) + "30"
url = "https://cs.sl.blackberry.com/cse/updateDetails/2.2/"
query = '<?xml version="1.0" encoding="UTF-8"?>'
query += '<updateDetailRequest version="2.2.1"'
query += ' authEchoTS="1366644680359">'
query += "<clientProperties>"
query += "<hardware>"
query += "<pin>0x2FFFFFB3</pin><bsn>1128121361</bsn>"
query += "<imei>004401139269240</imei>"
query += "<id>0x" + device + "</id>"
query += "</hardware>"
query += "<network>"
query += "<homeNPC>0x" + npc + "</homeNPC>"
query += "<iccid>89014104255505565333</iccid>"
query += "</network>"
query += "<software>"
query += "<currentLocale>en_US</currentLocale>"
query += "<legalLocale>en_US</legalLocale>"
query += "</software>"
query += "</clientProperties>"
query += "<updateDirectives>"
query += '<allowPatching type="REDBEND">true</allowPatching>'
query += "<upgradeMode>" + upg + "</upgradeMode>"
query += "<provideDescriptions>false</provideDescriptions>"
query += "<provideFiles>true</provideFiles>"
query += "<queryType>NOTIFICATION_CHECK</queryType>"
query += "</updateDirectives>"
query += "<pollType>manual</pollType>"
query += "<resultPackageSetCriteria>"
query += '<softwareRelease softwareReleaseVersion="latest" />'
query += "<releaseIndependent>"
query += '<packageType operation="include">application</packageType>'
query += "</releaseIndependent>"
query += "</resultPackageSetCriteria>"
query += "</updateDetailRequest>"
header = {"Content-Type": "text/xml;charset=UTF-8"}
r = requests.post(url, headers=header, data=query)
root = xml.etree.ElementTree.fromstring(r.text)
sw_exists = root.find('./data/content/softwareReleaseMetadata')
swver = ""
if sw_exists is None:
swver = "N/A"
else:
for child in root.iter("softwareReleaseMetadata"):
swver = child.get("softwareReleaseVersion")
files = []
osver = ""
radver = ""
package_exists = root.find('./data/content/fileSets/fileSet')
if package_exists is not None:
baseurl = package_exists.get("url") + "/"
for child in root.iter("package"):
if not blitz:
files.append(baseurl + child.get("path"))
else:
if child.get("type") not in ["system:radio",
"system:desktop",
"system:os"]:
files.append(baseurl + child.get("path"))
if child.get("type") == "system:radio":
radver = child.get("version")
if child.get("type") == "system:desktop":
osver = child.get("version")
if child.get("type") == "system:os":
osver = child.get("version")
else:
pass
return(swver, osver, radver, files)