PKvG^O++kad/__init__.pyfrom .kad import DHT __version__ = "0.5.5" PKaENG펪kad/bucketset.pyimport heapq import threading from .peer import Peer def largest_differing_bit(value1, value2): distance = value1 ^ value2 length = -1 while (distance): distance >>= 1 length += 1 return max(0, length) class BucketSet(object): def __init__(self, bucket_size, buckets, id): self.id = id self.bucket_size = bucket_size self.buckets = [list() for _ in range(buckets)] self.lock = threading.Lock() def to_list (self): l = [] for bucket in self.buckets: l += bucket return l def to_dict (self): l = [] for bucket in self.buckets: for peer in bucket: if len (peer) == 4: l.append ({'host': peer[0], 'port': peer[1], 'id': peer[2], 'info': peer[3]}) return l def insert(self, peer): if peer.id != self.id: bucket_number = largest_differing_bit(self.id, peer.id) peer_triple = peer.astriple() with self.lock: bucket = self.buckets[bucket_number] if peer_triple in bucket: bucket.pop(bucket.index(peer_triple)) elif len(bucket) >= self.bucket_size: bucket.pop(0) bucket.append(peer_triple) def nearest_nodes(self, key, limit=None): num_results = limit if limit else self.bucket_size with self.lock: def keyfunction(peer): return key ^ peer[2] # ideally there would be a better way with names? Instead of storing triples it would be nice to have a dict peers = (peer for bucket in self.buckets for peer in bucket) best_peers = heapq.nsmallest(self.bucket_size, peers, keyfunction) return [Peer(*peer) for peer in best_peers] PKP'G Nkad/shortlist.pyimport threading from .peer import Peer class Shortlist(object): def __init__(self, k, key): self.k = k self.key = key self.list = list() self.lock = threading.Lock() self.completion_value = None def set_complete(self, value): with self.lock: self.completion_value = value def completion_result(self): with self.lock: return self.completion_value def update(self, nodes): for node in nodes: self._update_one(node) def _update_one(self, node): if node.id == self.key or self.completion_value: return with self.lock: for i in range(len(self.list)): if node.id == self.list[i][0][2]: break if node.id ^ self.key < self.list[i][0][2] ^ self.key: self.list.insert(i, (node.astriple(), False)) self.list = self.list[:self.k] break else: if len(self.list) < self.k: self.list.append((node.astriple(), False)) def mark(self, node): with self.lock: for i in range(len(self.list)): if node.id == self.list[i][0][2]: self.list[i] = (node.astriple(), True) def complete(self): if self.completion_value: return True with self.lock: for node, completed in self.list: if not completed: return False return True def get_next_iteration(self, alpha): if self.completion_value: return [] next_iteration = [] with self.lock: for node, completed in self.list: if not completed: next_iteration.append(Peer(*node)) if len(next_iteration) >= alpha: break return next_iteration def results(self): with self.lock: return [Peer(*node) for (node, completed) in self.list]PKkDNG@&Y Y kad/peer.pyimport hashlib import json from .hashing import hash_function class Peer(object): ''' DHT Peer Information''' def __init__(self, host, port, id, info): self.host, self.port, self.id, self.info = host, port, id, info def astriple(self): return (self.host, self.port, self.id, self.info) def asquad(self): return (self.host, self.port, self.id, self.info) def address(self): return (self.host, self.port) def __repr__(self): return repr(self.astriple()) def _sendmessage(self, message, sock=None, peer_id=None, peer_info=None, lock=None): message["peer_id"] = peer_id # more like sender_id message["peer_info"] = peer_info encoded = json.dumps(message) if sock: if lock: with lock: sock.sendto(encoded.encode ('ascii'), (self.host, self.port)) else: sock.sendto(encoded.encode ('ascii'), (self.host, self.port)) def ping(self, socket=None, peer_id=None, peer_info=None, lock=None): message = { "message_type": "ping" } self._sendmessage(message, socket, peer_id=peer_id, peer_info=peer_info, lock=lock) def pong(self, socket=None, peer_id=None, peer_info=None, lock=None): message = { "message_type": "pong" } self._sendmessage(message, socket, peer_id=peer_id, peer_info=peer_info, lock=lock) def store(self, key, value, socket=None, peer_id=None, peer_info=None, lock=None): message = { "message_type": "store", "id": key, "value": value } self._sendmessage(message, socket, peer_id=peer_id, peer_info=peer_info, lock=lock) def find_node(self, id, rpc_id, socket=None, peer_id=None, peer_info=None, lock=None): message = { "message_type": "find_node", "id": id, "rpc_id": rpc_id } self._sendmessage(message, socket, peer_id=peer_id, peer_info=peer_info, lock=lock) def found_nodes(self, id, nearest_nodes, rpc_id, socket=None, peer_id=None, peer_info=None, lock=None): message = { "message_type": "found_nodes", "id": id, "nearest_nodes": nearest_nodes, "rpc_id": rpc_id } self._sendmessage(message, socket, peer_id=peer_id, peer_info=peer_info, lock=lock) def find_value(self, id, rpc_id, socket=None, peer_id=None, peer_info=None, lock=None): message = { "message_type": "find_value", "id": id, "rpc_id": rpc_id } self._sendmessage(message, socket, peer_id=peer_id, peer_info=peer_info, lock=lock) def found_value(self, id, value, rpc_id, socket=None, peer_id=None, peer_info=None, lock=None): message = { "message_type": "found_value", "id": id, "value": value, "rpc_id": rpc_id } self._sendmessage(message, socket, peer_id=peer_id, peer_info=peer_info, lock=lock) PKP'G7Jkad/storage.pyimport shelve def test (): pass class Shelve: def __init__ (self, f): self.shelve = shelve.open (f) def dump (self): for x in self.shelve: print ('key:',x,'\t\tvalue:',self.shelve[x]) def __getitem__(self, key): return self.shelve[str (key)] def __setitem__(self, key, value): self.shelve[str (key)] = value def __contains__(self, key): return str(key) in self.shelve PKvG/%Z@@ kad/kad.pyimport json import random import socket import socketserver import threading import time from .bucketset import BucketSet from .hashing import hash_function, random_id from .peer import Peer from .storage import Shelve from .shortlist import Shortlist from . import hashing k = 20 alpha = 3 id_bits = 128 iteration_sleep = 1 class DHTRequestHandler(socketserver.BaseRequestHandler): def handle(self): try: message = json.loads(self.request[0].decode ('utf-8').strip()) message_type = message["message_type"] if message_type == "ping": self.handle_ping(message) elif message_type == "pong": self.handle_pong(message) elif message_type == "find_node": self.handle_find(message) elif message_type == "find_value": self.handle_find(message, find_value=True) elif message_type == "found_nodes": self.handle_found_nodes(message) elif message_type == "found_value": self.handle_found_value(message) elif message_type == "store": self.handle_store(message) except KeyError: pass except ValueError: pass client_host, client_port = self.client_address peer_id = message["peer_id"] peer_info = message["peer_info"] new_peer = Peer(client_host, client_port, peer_id, peer_info) self.server.dht.buckets.insert(new_peer) def handle_ping(self, message): client_host, client_port = self.client_address id = message["peer_id"] info = message["peer_info"] peer = Peer(client_host, client_port, id, info) peer.pong(socket=self.server.socket, peer_id=self.server.dht.peer.id, lock=self.server.send_lock) def handle_pong(self, message): pass def handle_find(self, message, find_value=False): key = message["id"] id = message["peer_id"] info = message["peer_info"] client_host, client_port = self.client_address peer = Peer(client_host, client_port, id, info) response_socket = self.request[1] if find_value and (key in self.server.dht.data): value = self.server.dht.data[key] peer.found_value(id, value, message["rpc_id"], socket=response_socket, peer_id=self.server.dht.peer.id, peer_info=self.server.dht.peer.info, lock=self.server.send_lock) else: nearest_nodes = self.server.dht.buckets.nearest_nodes(id) if not nearest_nodes: nearest_nodes.append(self.server.dht.peer) nearest_nodes = [nearest_peer.astriple() for nearest_peer in nearest_nodes] peer.found_nodes(id, nearest_nodes, message["rpc_id"], socket=response_socket, peer_id=self.server.dht.peer.id, peer_info=self.server.dht.peer.info, lock=self.server.send_lock) def handle_found_nodes(self, message): rpc_id = message["rpc_id"] shortlist = self.server.dht.rpc_ids[rpc_id] del self.server.dht.rpc_ids[rpc_id] nearest_nodes = [Peer(*peer) for peer in message["nearest_nodes"]] shortlist.update(nearest_nodes) def handle_found_value(self, message): rpc_id = message["rpc_id"] shortlist = self.server.dht.rpc_ids[rpc_id] del self.server.dht.rpc_ids[rpc_id] shortlist.set_complete(message["value"]) def handle_store(self, message): key = message["id"] self.server.dht.data[key] = message["value"] class DHTServer(socketserver.ThreadingMixIn, socketserver.UDPServer): def __init__(self, host_address, handler_cls): socketserver.UDPServer.__init__(self, host_address, handler_cls) self.send_lock = threading.Lock() class DHT(object): def __init__(self, host, port, id=None, seeds=[], storage={}, info={}, requesthandler=DHTRequestHandler): if not id: id = random_id() self.storage = storage self.info = info self.hash_function = hashing.hash_function self.peer = Peer(host, port, id, info) self.data = self.storage self.buckets = BucketSet(k, id_bits, self.peer.id) self.rpc_ids = {} # should probably have a lock for this self.server = DHTServer (self.peer.address(), requesthandler) self.server.dht = self self.server_thread = threading.Thread(target=self.server.serve_forever) self.server_thread.daemon = True self.server_thread.start() self.bootstrap (seeds) def identity (self): return self.peer.id def iterative_find_nodes(self, key, boot_peer=None): shortlist = Shortlist(k, key) shortlist.update(self.buckets.nearest_nodes(key, limit=alpha)) if boot_peer: rpc_id = random.getrandbits(id_bits) self.rpc_ids[rpc_id] = shortlist boot_peer.find_node(key, rpc_id, socket=self.server.socket, peer_id=self.peer.id, peer_info=self.peer.info) while (not shortlist.complete()) or boot_peer: nearest_nodes = shortlist.get_next_iteration(alpha) for peer in nearest_nodes: shortlist.mark(peer) rpc_id = random.getrandbits(id_bits) self.rpc_ids[rpc_id] = shortlist peer.find_node(key, rpc_id, socket=self.server.socket, peer_id=self.peer.id, peer_info=self.info) ###### time.sleep(iteration_sleep) boot_peer = None return shortlist.results() def iterative_find_value(self, key): shortlist = Shortlist(k, key) shortlist.update(self.buckets.nearest_nodes(key, limit=alpha)) while not shortlist.complete(): nearest_nodes = shortlist.get_next_iteration(alpha) for peer in nearest_nodes: shortlist.mark(peer) rpc_id = random.getrandbits(id_bits) self.rpc_ids[rpc_id] = shortlist peer.find_value(key, rpc_id, socket=self.server.socket, peer_id=self.peer.id, peer_info=self.info) ##### time.sleep(iteration_sleep) return shortlist.completion_result() # Return the list of connected peers def peers (self): return self.buckets.to_dict () # Boostrap the network with a list of bootstrap nodes def bootstrap(self, bootstrap_nodes = []): for bnode in bootstrap_nodes: boot_peer = Peer(bnode[0], bnode[1], "", "") self.iterative_find_nodes(self.peer.id, boot_peer=boot_peer) if len (bootstrap_nodes) == 0: for bnode in self.buckets.to_list (): self.iterative_find_nodes(self.peer.id, boot_peer=Peer (bnode[0], bnode[1], bnode[2], bnode[3])) # Get a value in a sync way, calling an handler def get_sync (self, key, handler): try: d = self[key] except: d = None handler (d) # Get a value in async way def get (self, key, handler): #print ('dht.get',key) t = threading.Thread(target=self.get_sync, args=(key, handler)) t.start () # Iterator def __iter__ (self): return self.data.__iter__ () # Operator [] def __getitem__(self, key): if type (key) == int: hashed_key = key else: hashed_key = self.hash_function (key) if hashed_key in self.data: return self.data[hashed_key] result = self.iterative_find_value(hashed_key) if result: return result raise KeyError # Operator []= def __setitem__(self, key, value): hashed_key = self.hash_function (key) #print ('dht.set',key,value,hashed_key) nearest_nodes = self.iterative_find_nodes(hashed_key) if not nearest_nodes: self.data[hashed_key] = value for node in nearest_nodes: node.store(hashed_key, value, socket=self.server.socket, peer_id=self.peer.id) def tick(): pass PKP'Gzkad/hashing.pyimport hashlib import random id_bits = 128 def hash_function (data): return int(hashlib.md5(data.encode ('ascii')).hexdigest(), 16) def random_id (seed=None): if seed: random.seed(seed) return random.randint(0, (2 ** id_bits)-1) PK9vGr&kad.py-0.5.5.dist-info/DESCRIPTION.rstCopyright (c) 2015, Davide Gessa All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. The views and conclusions contained in the software and documentation are those of the authors and should not be interpreted as representing official policies, either expressed or implied, of the FreeBSD Project. Description: # kad.py Python3 implementation of the Kademlia DHT data store. Useful for distributing a key-value store in a decentralized manner. To create a new DHT swarm, just call DHT() with the host and port that you will listen on. To join an existing DHT swarm, also provide bootstrap host and port numbers of any existing node. The nodes will discover the rest of the swarm as appropriate during usage. ## Example: A two-node DHT ```python from kad import DHT host1, port1 = 'localhost', 3000 dht1 = DHT(host1, port1) host2, port2 = 'localhost', 3001 dht2 = DHT(host2, port2, seeds=[(host1, port1)]) dht1["my_key"] = [u"My", u"json-serializable", u"Object"] print (dht2["my_key"]) # blocking get dht2.get ("my_key", lambda data: print (data)) # threaded get ``` ## Example: Persistent storage We can use a custom storage for local data. Storage parameter must an object with __getitem__ and __setitem__. In this example we use shelve to create a persistent storage. ```python from kad import DHT import shelve host, port = 'localhost', 3000 dht = DHT(host, port, storage=shelve.open ('sto.dat')) ``` ## Example: Custom hash function By default, kad.py doesn't hash keys. We can provide a custom hash_function. ```python from kad import DHT host, port = 'localhost', 3000 dht = DHT(host, port, hash_function=lambda d: d[0:4]) ``` ## Example: Custom request handler You can extend the default DHTRequestHandler to intercept any kind of messages. ```python from kad import * class CustomRequestHandler (kad.DHTRequestHandler): def handle_store(self, message): print (message['value']) return super (CustomRequestHandler, self).handle_store (message) d = DHT ('localhost', 3030, requesthandler=CustomRequestHandler) d['ciao'] = {'hola': 12} ``` ## Example: Iterate over DHT keys You can use the DHT object as iterator for stored keys. ```python from kad import DHT d = DHT ('localhost', 3100) d['ciao'] = 'mondo' d['hello'] = 'world' for key in d: print (key, d[key]) ``` Platform: UNKNOWN PK9vGe~ $kad.py-0.5.5.dist-info/metadata.json{"extensions": {"python.details": {"contacts": [{"email": "gessadavide@gmail.com, isaac@zafuta.com", "name": "Davide Gessa, Isaac Zafuta", "role": "author"}], "document_names": {"description": "DESCRIPTION.rst"}, "project_urls": {"Home": "https://github.com/dakk/kad.py"}}}, "generator": "bdist_wheel (0.26.0)", "license": "Copyright (c) 2012, Isaac Zafuta", "metadata_version": "2.0", "name": "kad.py", "summary": "Python3 DHT Implementation", "version": "0.5.5"}PK9vGI2e$kad.py-0.5.5.dist-info/top_level.txtkad PK9vG}\\kad.py-0.5.5.dist-info/WHEELWheel-Version: 1.0 Generator: bdist_wheel (0.26.0) Root-Is-Purelib: true Tag: py3-none-any PK9vG,kad.py-0.5.5.dist-info/METADATAMetadata-Version: 2.0 Name: kad.py Version: 0.5.5 Summary: Python3 DHT Implementation Home-page: https://github.com/dakk/kad.py Author: Davide Gessa, Isaac Zafuta Author-email: gessadavide@gmail.com, isaac@zafuta.com License: Copyright (c) 2012, Isaac Zafuta Copyright (c) 2015, Davide Gessa All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. The views and conclusions contained in the software and documentation are those of the authors and should not be interpreted as representing official policies, either expressed or implied, of the FreeBSD Project. Description: # kad.py Python3 implementation of the Kademlia DHT data store. Useful for distributing a key-value store in a decentralized manner. To create a new DHT swarm, just call DHT() with the host and port that you will listen on. To join an existing DHT swarm, also provide bootstrap host and port numbers of any existing node. The nodes will discover the rest of the swarm as appropriate during usage. ## Example: A two-node DHT ```python from kad import DHT host1, port1 = 'localhost', 3000 dht1 = DHT(host1, port1) host2, port2 = 'localhost', 3001 dht2 = DHT(host2, port2, seeds=[(host1, port1)]) dht1["my_key"] = [u"My", u"json-serializable", u"Object"] print (dht2["my_key"]) # blocking get dht2.get ("my_key", lambda data: print (data)) # threaded get ``` ## Example: Persistent storage We can use a custom storage for local data. Storage parameter must an object with __getitem__ and __setitem__. In this example we use shelve to create a persistent storage. ```python from kad import DHT import shelve host, port = 'localhost', 3000 dht = DHT(host, port, storage=shelve.open ('sto.dat')) ``` ## Example: Custom hash function By default, kad.py doesn't hash keys. We can provide a custom hash_function. ```python from kad import DHT host, port = 'localhost', 3000 dht = DHT(host, port, hash_function=lambda d: d[0:4]) ``` ## Example: Custom request handler You can extend the default DHTRequestHandler to intercept any kind of messages. ```python from kad import * class CustomRequestHandler (kad.DHTRequestHandler): def handle_store(self, message): print (message['value']) return super (CustomRequestHandler, self).handle_store (message) d = DHT ('localhost', 3030, requesthandler=CustomRequestHandler) d['ciao'] = {'hola': 12} ``` ## Example: Iterate over DHT keys You can use the DHT object as iterator for stored keys. ```python from kad import DHT d = DHT ('localhost', 3100) d['ciao'] = 'mondo' d['hello'] = 'world' for key in d: print (key, d[key]) ``` Platform: UNKNOWN PK9vG w/kad.py-0.5.5.dist-info/RECORDkad/__init__.py,sha256=jhZPbAyhJKKd79gJgnH3mHTWFqh3wD5kd2yzQSXVrEU,43 kad/bucketset.py,sha256=inE86PLQOOmv3WsLGL6JEiWfsuaY-bmZLKE0CYaryAA,1531 kad/hashing.py,sha256=a7QtlgHsT93QcVq0p09HsybvcgdccxT6lV9FlUvsAw8,261 kad/kad.py,sha256=hD0_IHeRX5zXvUwmwwQo3mFeybO-_N5zuZ1uYfATcgI,6976 kad/peer.py,sha256=Cnbd8U7oDK8P8k3waOLdqXSmEcrqkWdjQZgxiEmeao0,3161 kad/shortlist.py,sha256=0JCqjDb7hBehbDmiYou51w_pJKcO6Pyb9LLrk6NuHrQ,2185 kad/storage.py,sha256=oR-gcmeQhFFKPmnS7OX9vq67-t4pDMYQJIFqkQ0aaww,398 kad.py-0.5.5.dist-info/DESCRIPTION.rst,sha256=1wHOdr99dcpDb4iJV3o812GorT6ZFOpmE3fMBxtEdh8,4257 kad.py-0.5.5.dist-info/METADATA,sha256=3CYnFzVq6MTkdGFykPYyp1232rAIyFNZj6x8k2-2Tig,4517 kad.py-0.5.5.dist-info/RECORD,, kad.py-0.5.5.dist-info/WHEEL,sha256=zX7PHtH_7K-lEzyK75et0UBa3Bj8egCBMXe1M4gc6SU,92 kad.py-0.5.5.dist-info/metadata.json,sha256=ywXZwnEYw963vrA-FGysVTq6-F8blOvtXuc9d3EIPW4,464 kad.py-0.5.5.dist-info/top_level.txt,sha256=9cpEq7knru3jBirZCsp4W8yh4qZ3i-f3a90qMGIJ0Fc,4 PKvG^O++kad/__init__.pyPKaENG펪Xkad/bucketset.pyPKP'G Nkad/shortlist.pyPKkDNG@&Y Y 8kad/peer.pyPKP'G7Jkad/storage.pyPKvG/%Z@@ tkad/kad.pyPKP'Gz8kad/hashing.pyPK9vGr& :kad.py-0.5.5.dist-info/DESCRIPTION.rstPK9vGe~ $Jkad.py-0.5.5.dist-info/metadata.jsonPK9vGI2e$Mkad.py-0.5.5.dist-info/top_level.txtPK9vG}\\JMkad.py-0.5.5.dist-info/WHEELPK9vG,Mkad.py-0.5.5.dist-info/METADATAPK9vG w/_kad.py-0.5.5.dist-info/RECORDPK |c