���ѧۧݧ�ӧ�� �ާ֧ߧ֧էا֧� - ���֧էѧܧ�ڧ��ӧѧ�� - /home/ukubnwwtacc0unt/chapelbellstudios.com/uploads/cover/multiprocessing.tar
���ѧ٧ѧ�
__init__.py 0000644 00000001633 15204243001 0006647 0 ustar 00 # # Package analogous to 'threading.py' but using processes # # multiprocessing/__init__.py # # This package is intended to duplicate the functionality (and much of # the API) of threading.py but uses processes instead of threads. A # subpackage 'multiprocessing.dummy' has the same API but is a simple # wrapper for 'threading'. # # Copyright (c) 2006-2008, R Oudkerk # Licensed to PSF under a Contributor Agreement. # import sys from . import context # # Copy stuff from default context # globals().update((name, getattr(context._default_context, name)) for name in context._default_context.__all__) __all__ = context._default_context.__all__ # # XXX These should not really be documented or public. # SUBDEBUG = 5 SUBWARNING = 25 # # Alias for main module -- will be reset by bootstrapping child processes # if '__main__' in sys.modules: sys.modules['__mp_main__'] = sys.modules['__main__'] context.py 0000644 00000025245 15204243001 0006601 0 ustar 00 import os import sys import threading from . import process from . import reduction __all__ = [] # things are copied from here to __init__.py # # Exceptions # class ProcessError(Exception): pass class BufferTooShort(ProcessError): pass class TimeoutError(ProcessError): pass class AuthenticationError(ProcessError): pass # # Base type for contexts # class BaseContext(object): ProcessError = ProcessError BufferTooShort = BufferTooShort TimeoutError = TimeoutError AuthenticationError = AuthenticationError current_process = staticmethod(process.current_process) active_children = staticmethod(process.active_children) def cpu_count(self): '''Returns the number of CPUs in the system''' num = os.cpu_count() if num is None: raise NotImplementedError('cannot determine number of cpus') else: return num def Manager(self): '''Returns a manager associated with a running server process The managers methods such as `Lock()`, `Condition()` and `Queue()` can be used to create shared objects. ''' from .managers import SyncManager m = SyncManager(ctx=self.get_context()) m.start() return m def Pipe(self, duplex=True): '''Returns two connection object connected by a pipe''' from .connection import Pipe return Pipe(duplex) def Lock(self): '''Returns a non-recursive lock object''' from .synchronize import Lock return Lock(ctx=self.get_context()) def RLock(self): '''Returns a recursive lock object''' from .synchronize import RLock return RLock(ctx=self.get_context()) def Condition(self, lock=None): '''Returns a condition object''' from .synchronize import Condition return Condition(lock, ctx=self.get_context()) def Semaphore(self, value=1): '''Returns a semaphore object''' from .synchronize import Semaphore return Semaphore(value, ctx=self.get_context()) def BoundedSemaphore(self, value=1): '''Returns a bounded semaphore object''' from .synchronize import BoundedSemaphore return BoundedSemaphore(value, ctx=self.get_context()) def Event(self): '''Returns an event object''' from .synchronize import Event return Event(ctx=self.get_context()) def Barrier(self, parties, action=None, timeout=None): '''Returns a barrier object''' from .synchronize import Barrier return Barrier(parties, action, timeout, ctx=self.get_context()) def Queue(self, maxsize=0): '''Returns a queue object''' from .queues import Queue return Queue(maxsize, ctx=self.get_context()) def JoinableQueue(self, maxsize=0): '''Returns a queue object''' from .queues import JoinableQueue return JoinableQueue(maxsize, ctx=self.get_context()) def SimpleQueue(self): '''Returns a queue object''' from .queues import SimpleQueue return SimpleQueue(ctx=self.get_context()) def Pool(self, processes=None, initializer=None, initargs=(), maxtasksperchild=None): '''Returns a process pool object''' from .pool import Pool return Pool(processes, initializer, initargs, maxtasksperchild, context=self.get_context()) def RawValue(self, typecode_or_type, *args): '''Returns a shared object''' from .sharedctypes import RawValue return RawValue(typecode_or_type, *args) def RawArray(self, typecode_or_type, size_or_initializer): '''Returns a shared array''' from .sharedctypes import RawArray return RawArray(typecode_or_type, size_or_initializer) def Value(self, typecode_or_type, *args, lock=True): '''Returns a synchronized shared object''' from .sharedctypes import Value return Value(typecode_or_type, *args, lock=lock, ctx=self.get_context()) def Array(self, typecode_or_type, size_or_initializer, *, lock=True): '''Returns a synchronized shared array''' from .sharedctypes import Array return Array(typecode_or_type, size_or_initializer, lock=lock, ctx=self.get_context()) def freeze_support(self): '''Check whether this is a fake forked process in a frozen executable. If so then run code specified by commandline and exit. ''' if sys.platform == 'win32' and getattr(sys, 'frozen', False): from .spawn import freeze_support freeze_support() def get_logger(self): '''Return package logger -- if it does not already exist then it is created. ''' from .util import get_logger return get_logger() def log_to_stderr(self, level=None): '''Turn on logging and add a handler which prints to stderr''' from .util import log_to_stderr return log_to_stderr(level) def allow_connection_pickling(self): '''Install support for sending connections and sockets between processes ''' # This is undocumented. In previous versions of multiprocessing # its only effect was to make socket objects inheritable on Windows. from . import connection def set_executable(self, executable): '''Sets the path to a python.exe or pythonw.exe binary used to run child processes instead of sys.executable when using the 'spawn' start method. Useful for people embedding Python. ''' from .spawn import set_executable set_executable(executable) def set_forkserver_preload(self, module_names): '''Set list of module names to try to load in forkserver process. This is really just a hint. ''' from .forkserver import set_forkserver_preload set_forkserver_preload(module_names) def get_context(self, method=None): if method is None: return self try: ctx = _concrete_contexts[method] except KeyError: raise ValueError('cannot find context for %r' % method) ctx._check_available() return ctx def get_start_method(self, allow_none=False): return self._name def set_start_method(self, method, force=False): raise ValueError('cannot set start method of concrete context') @property def reducer(self): '''Controls how objects will be reduced to a form that can be shared with other processes.''' return globals().get('reduction') @reducer.setter def reducer(self, reduction): globals()['reduction'] = reduction def _check_available(self): pass # # Type of default context -- underlying context can be set at most once # class Process(process.BaseProcess): _start_method = None @staticmethod def _Popen(process_obj): return _default_context.get_context().Process._Popen(process_obj) class DefaultContext(BaseContext): Process = Process def __init__(self, context): self._default_context = context self._actual_context = None def get_context(self, method=None): if method is None: if self._actual_context is None: self._actual_context = self._default_context return self._actual_context else: return super().get_context(method) def set_start_method(self, method, force=False): if self._actual_context is not None and not force: raise RuntimeError('context has already been set') if method is None and force: self._actual_context = None return self._actual_context = self.get_context(method) def get_start_method(self, allow_none=False): if self._actual_context is None: if allow_none: return None self._actual_context = self._default_context return self._actual_context._name def get_all_start_methods(self): if sys.platform == 'win32': return ['spawn'] else: if reduction.HAVE_SEND_HANDLE: return ['fork', 'spawn', 'forkserver'] else: return ['fork', 'spawn'] DefaultContext.__all__ = list(x for x in dir(DefaultContext) if x[0] != '_') # # Context types for fixed start method # if sys.platform != 'win32': class ForkProcess(process.BaseProcess): _start_method = 'fork' @staticmethod def _Popen(process_obj): from .popen_fork import Popen return Popen(process_obj) class SpawnProcess(process.BaseProcess): _start_method = 'spawn' @staticmethod def _Popen(process_obj): from .popen_spawn_posix import Popen return Popen(process_obj) class ForkServerProcess(process.BaseProcess): _start_method = 'forkserver' @staticmethod def _Popen(process_obj): from .popen_forkserver import Popen return Popen(process_obj) class ForkContext(BaseContext): _name = 'fork' Process = ForkProcess class SpawnContext(BaseContext): _name = 'spawn' Process = SpawnProcess class ForkServerContext(BaseContext): _name = 'forkserver' Process = ForkServerProcess def _check_available(self): if not reduction.HAVE_SEND_HANDLE: raise ValueError('forkserver start method not available') _concrete_contexts = { 'fork': ForkContext(), 'spawn': SpawnContext(), 'forkserver': ForkServerContext(), } _default_context = DefaultContext(_concrete_contexts['fork']) else: class SpawnProcess(process.BaseProcess): _start_method = 'spawn' @staticmethod def _Popen(process_obj): from .popen_spawn_win32 import Popen return Popen(process_obj) class SpawnContext(BaseContext): _name = 'spawn' Process = SpawnProcess _concrete_contexts = { 'spawn': SpawnContext(), } _default_context = DefaultContext(_concrete_contexts['spawn']) # # Force the start method # def _force_start_method(method): _default_context._actual_context = _concrete_contexts[method] # # Check that the current thread is spawning a child process # _tls = threading.local() def get_spawning_popen(): return getattr(_tls, 'spawning_popen', None) def set_spawning_popen(popen): _tls.spawning_popen = popen def assert_spawning(obj): if get_spawning_popen() is None: raise RuntimeError( '%s objects should only be shared between processes' ' through inheritance' % type(obj).__name__ ) pool.py 0000644 00000062713 15204243001 0006067 0 ustar 00 # # Module providing the `Pool` class for managing a process pool # # multiprocessing/pool.py # # Copyright (c) 2006-2008, R Oudkerk # Licensed to PSF under a Contributor Agreement. # __all__ = ['Pool', 'ThreadPool'] # # Imports # import threading import queue import itertools import collections import os import time import traceback # If threading is available then ThreadPool should be provided. Therefore # we avoid top-level imports which are liable to fail on some systems. from . import util from . import get_context, TimeoutError # # Constants representing the state of a pool # RUN = 0 CLOSE = 1 TERMINATE = 2 # # Miscellaneous # job_counter = itertools.count() def mapstar(args): return list(map(*args)) def starmapstar(args): return list(itertools.starmap(args[0], args[1])) # # Hack to embed stringification of remote traceback in local traceback # class RemoteTraceback(Exception): def __init__(self, tb): self.tb = tb def __str__(self): return self.tb class ExceptionWithTraceback: def __init__(self, exc, tb): tb = traceback.format_exception(type(exc), exc, tb) tb = ''.join(tb) self.exc = exc self.tb = '\n"""\n%s"""' % tb def __reduce__(self): return rebuild_exc, (self.exc, self.tb) def rebuild_exc(exc, tb): exc.__cause__ = RemoteTraceback(tb) return exc # # Code run by worker processes # class MaybeEncodingError(Exception): """Wraps possible unpickleable errors, so they can be safely sent through the socket.""" def __init__(self, exc, value): self.exc = repr(exc) self.value = repr(value) super(MaybeEncodingError, self).__init__(self.exc, self.value) def __str__(self): return "Error sending result: '%s'. Reason: '%s'" % (self.value, self.exc) def __repr__(self): return "<%s: %s>" % (self.__class__.__name__, self) def worker(inqueue, outqueue, initializer=None, initargs=(), maxtasks=None, wrap_exception=False): assert maxtasks is None or (type(maxtasks) == int and maxtasks > 0) put = outqueue.put get = inqueue.get if hasattr(inqueue, '_writer'): inqueue._writer.close() outqueue._reader.close() if initializer is not None: initializer(*initargs) completed = 0 while maxtasks is None or (maxtasks and completed < maxtasks): try: task = get() except (EOFError, OSError): util.debug('worker got EOFError or OSError -- exiting') break if task is None: util.debug('worker got sentinel -- exiting') break job, i, func, args, kwds = task try: result = (True, func(*args, **kwds)) except Exception as e: if wrap_exception and func is not _helper_reraises_exception: e = ExceptionWithTraceback(e, e.__traceback__) result = (False, e) try: put((job, i, result)) except Exception as e: wrapped = MaybeEncodingError(e, result[1]) util.debug("Possible encoding error while sending result: %s" % ( wrapped)) put((job, i, (False, wrapped))) task = job = result = func = args = kwds = None completed += 1 util.debug('worker exiting after %d tasks' % completed) def _helper_reraises_exception(ex): 'Pickle-able helper function for use by _guarded_task_generation.' raise ex # # Class representing a process pool # class Pool(object): ''' Class which supports an async version of applying functions to arguments. ''' _wrap_exception = True def Process(self, *args, **kwds): return self._ctx.Process(*args, **kwds) def __init__(self, processes=None, initializer=None, initargs=(), maxtasksperchild=None, context=None): self._ctx = context or get_context() self._setup_queues() self._taskqueue = queue.Queue() self._cache = {} self._state = RUN self._maxtasksperchild = maxtasksperchild self._initializer = initializer self._initargs = initargs if processes is None: processes = os.cpu_count() or 1 if processes < 1: raise ValueError("Number of processes must be at least 1") if initializer is not None and not callable(initializer): raise TypeError('initializer must be a callable') self._processes = processes self._pool = [] self._repopulate_pool() self._worker_handler = threading.Thread( target=Pool._handle_workers, args=(self, ) ) self._worker_handler.daemon = True self._worker_handler._state = RUN self._worker_handler.start() self._task_handler = threading.Thread( target=Pool._handle_tasks, args=(self._taskqueue, self._quick_put, self._outqueue, self._pool, self._cache) ) self._task_handler.daemon = True self._task_handler._state = RUN self._task_handler.start() self._result_handler = threading.Thread( target=Pool._handle_results, args=(self._outqueue, self._quick_get, self._cache) ) self._result_handler.daemon = True self._result_handler._state = RUN self._result_handler.start() self._terminate = util.Finalize( self, self._terminate_pool, args=(self._taskqueue, self._inqueue, self._outqueue, self._pool, self._worker_handler, self._task_handler, self._result_handler, self._cache), exitpriority=15 ) def _join_exited_workers(self): """Cleanup after any worker processes which have exited due to reaching their specified lifetime. Returns True if any workers were cleaned up. """ cleaned = False for i in reversed(range(len(self._pool))): worker = self._pool[i] if worker.exitcode is not None: # worker exited util.debug('cleaning up worker %d' % i) worker.join() cleaned = True del self._pool[i] return cleaned def _repopulate_pool(self): """Bring the number of pool processes up to the specified number, for use after reaping workers which have exited. """ for i in range(self._processes - len(self._pool)): w = self.Process(target=worker, args=(self._inqueue, self._outqueue, self._initializer, self._initargs, self._maxtasksperchild, self._wrap_exception) ) self._pool.append(w) w.name = w.name.replace('Process', 'PoolWorker') w.daemon = True w.start() util.debug('added worker') def _maintain_pool(self): """Clean up any exited workers and start replacements for them. """ if self._join_exited_workers(): self._repopulate_pool() def _setup_queues(self): self._inqueue = self._ctx.SimpleQueue() self._outqueue = self._ctx.SimpleQueue() self._quick_put = self._inqueue._writer.send self._quick_get = self._outqueue._reader.recv def apply(self, func, args=(), kwds={}): ''' Equivalent of `func(*args, **kwds)`. ''' assert self._state == RUN return self.apply_async(func, args, kwds).get() def map(self, func, iterable, chunksize=None): ''' Apply `func` to each element in `iterable`, collecting the results in a list that is returned. ''' return self._map_async(func, iterable, mapstar, chunksize).get() def starmap(self, func, iterable, chunksize=None): ''' Like `map()` method but the elements of the `iterable` are expected to be iterables as well and will be unpacked as arguments. Hence `func` and (a, b) becomes func(a, b). ''' return self._map_async(func, iterable, starmapstar, chunksize).get() def starmap_async(self, func, iterable, chunksize=None, callback=None, error_callback=None): ''' Asynchronous version of `starmap()` method. ''' return self._map_async(func, iterable, starmapstar, chunksize, callback, error_callback) def _guarded_task_generation(self, result_job, func, iterable): '''Provides a generator of tasks for imap and imap_unordered with appropriate handling for iterables which throw exceptions during iteration.''' try: i = -1 for i, x in enumerate(iterable): yield (result_job, i, func, (x,), {}) except Exception as e: yield (result_job, i+1, _helper_reraises_exception, (e,), {}) def imap(self, func, iterable, chunksize=1): ''' Equivalent of `map()` -- can be MUCH slower than `Pool.map()`. ''' if self._state != RUN: raise ValueError("Pool not running") if chunksize == 1: result = IMapIterator(self._cache) self._taskqueue.put( ( self._guarded_task_generation(result._job, func, iterable), result._set_length )) return result else: assert chunksize > 1 task_batches = Pool._get_tasks(func, iterable, chunksize) result = IMapIterator(self._cache) self._taskqueue.put( ( self._guarded_task_generation(result._job, mapstar, task_batches), result._set_length )) return (item for chunk in result for item in chunk) def imap_unordered(self, func, iterable, chunksize=1): ''' Like `imap()` method but ordering of results is arbitrary. ''' if self._state != RUN: raise ValueError("Pool not running") if chunksize == 1: result = IMapUnorderedIterator(self._cache) self._taskqueue.put( ( self._guarded_task_generation(result._job, func, iterable), result._set_length )) return result else: assert chunksize > 1 task_batches = Pool._get_tasks(func, iterable, chunksize) result = IMapUnorderedIterator(self._cache) self._taskqueue.put( ( self._guarded_task_generation(result._job, mapstar, task_batches), result._set_length )) return (item for chunk in result for item in chunk) def apply_async(self, func, args=(), kwds={}, callback=None, error_callback=None): ''' Asynchronous version of `apply()` method. ''' if self._state != RUN: raise ValueError("Pool not running") result = ApplyResult(self._cache, callback, error_callback) self._taskqueue.put(([(result._job, 0, func, args, kwds)], None)) return result def map_async(self, func, iterable, chunksize=None, callback=None, error_callback=None): ''' Asynchronous version of `map()` method. ''' return self._map_async(func, iterable, mapstar, chunksize, callback, error_callback) def _map_async(self, func, iterable, mapper, chunksize=None, callback=None, error_callback=None): ''' Helper function to implement map, starmap and their async counterparts. ''' if self._state != RUN: raise ValueError("Pool not running") if not hasattr(iterable, '__len__'): iterable = list(iterable) if chunksize is None: chunksize, extra = divmod(len(iterable), len(self._pool) * 4) if extra: chunksize += 1 if len(iterable) == 0: chunksize = 0 task_batches = Pool._get_tasks(func, iterable, chunksize) result = MapResult(self._cache, chunksize, len(iterable), callback, error_callback=error_callback) self._taskqueue.put( ( self._guarded_task_generation(result._job, mapper, task_batches), None ) ) return result @staticmethod def _handle_workers(pool): thread = threading.current_thread() # Keep maintaining workers until the cache gets drained, unless the pool # is terminated. while thread._state == RUN or (pool._cache and thread._state != TERMINATE): pool._maintain_pool() time.sleep(0.1) # send sentinel to stop workers pool._taskqueue.put(None) util.debug('worker handler exiting') @staticmethod def _handle_tasks(taskqueue, put, outqueue, pool, cache): thread = threading.current_thread() for taskseq, set_length in iter(taskqueue.get, None): task = None try: # iterating taskseq cannot fail for task in taskseq: if thread._state: util.debug('task handler found thread._state != RUN') break try: put(task) except Exception as e: job, idx = task[:2] try: cache[job]._set(idx, (False, e)) except KeyError: pass else: if set_length: util.debug('doing set_length()') idx = task[1] if task else -1 set_length(idx + 1) continue break finally: task = taskseq = job = None else: util.debug('task handler got sentinel') try: # tell result handler to finish when cache is empty util.debug('task handler sending sentinel to result handler') outqueue.put(None) # tell workers there is no more work util.debug('task handler sending sentinel to workers') for p in pool: put(None) except OSError: util.debug('task handler got OSError when sending sentinels') util.debug('task handler exiting') @staticmethod def _handle_results(outqueue, get, cache): thread = threading.current_thread() while 1: try: task = get() except (OSError, EOFError): util.debug('result handler got EOFError/OSError -- exiting') return if thread._state: assert thread._state == TERMINATE util.debug('result handler found thread._state=TERMINATE') break if task is None: util.debug('result handler got sentinel') break job, i, obj = task try: cache[job]._set(i, obj) except KeyError: pass task = job = obj = None while cache and thread._state != TERMINATE: try: task = get() except (OSError, EOFError): util.debug('result handler got EOFError/OSError -- exiting') return if task is None: util.debug('result handler ignoring extra sentinel') continue job, i, obj = task try: cache[job]._set(i, obj) except KeyError: pass task = job = obj = None if hasattr(outqueue, '_reader'): util.debug('ensuring that outqueue is not full') # If we don't make room available in outqueue then # attempts to add the sentinel (None) to outqueue may # block. There is guaranteed to be no more than 2 sentinels. try: for i in range(10): if not outqueue._reader.poll(): break get() except (OSError, EOFError): pass util.debug('result handler exiting: len(cache)=%s, thread._state=%s', len(cache), thread._state) @staticmethod def _get_tasks(func, it, size): it = iter(it) while 1: x = tuple(itertools.islice(it, size)) if not x: return yield (func, x) def __reduce__(self): raise NotImplementedError( 'pool objects cannot be passed between processes or pickled' ) def close(self): util.debug('closing pool') if self._state == RUN: self._state = CLOSE self._worker_handler._state = CLOSE def terminate(self): util.debug('terminating pool') self._state = TERMINATE self._worker_handler._state = TERMINATE self._terminate() def join(self): util.debug('joining pool') assert self._state in (CLOSE, TERMINATE) self._worker_handler.join() self._task_handler.join() self._result_handler.join() for p in self._pool: p.join() @staticmethod def _help_stuff_finish(inqueue, task_handler, size): # task_handler may be blocked trying to put items on inqueue util.debug('removing tasks from inqueue until task handler finished') inqueue._rlock.acquire() while task_handler.is_alive() and inqueue._reader.poll(): inqueue._reader.recv() time.sleep(0) @classmethod def _terminate_pool(cls, taskqueue, inqueue, outqueue, pool, worker_handler, task_handler, result_handler, cache): # this is guaranteed to only be called once util.debug('finalizing pool') worker_handler._state = TERMINATE task_handler._state = TERMINATE util.debug('helping task handler/workers to finish') cls._help_stuff_finish(inqueue, task_handler, len(pool)) assert result_handler.is_alive() or len(cache) == 0 result_handler._state = TERMINATE outqueue.put(None) # sentinel # We must wait for the worker handler to exit before terminating # workers because we don't want workers to be restarted behind our back. util.debug('joining worker handler') if threading.current_thread() is not worker_handler: worker_handler.join() # Terminate workers which haven't already finished. if pool and hasattr(pool[0], 'terminate'): util.debug('terminating workers') for p in pool: if p.exitcode is None: p.terminate() util.debug('joining task handler') if threading.current_thread() is not task_handler: task_handler.join() util.debug('joining result handler') if threading.current_thread() is not result_handler: result_handler.join() if pool and hasattr(pool[0], 'terminate'): util.debug('joining pool workers') for p in pool: if p.is_alive(): # worker has not yet exited util.debug('cleaning up worker %d' % p.pid) p.join() def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.terminate() # # Class whose instances are returned by `Pool.apply_async()` # class ApplyResult(object): def __init__(self, cache, callback, error_callback): self._event = threading.Event() self._job = next(job_counter) self._cache = cache self._callback = callback self._error_callback = error_callback cache[self._job] = self def ready(self): return self._event.is_set() def successful(self): assert self.ready() return self._success def wait(self, timeout=None): self._event.wait(timeout) def get(self, timeout=None): self.wait(timeout) if not self.ready(): raise TimeoutError if self._success: return self._value else: raise self._value def _set(self, i, obj): self._success, self._value = obj if self._callback and self._success: self._callback(self._value) if self._error_callback and not self._success: self._error_callback(self._value) self._event.set() del self._cache[self._job] AsyncResult = ApplyResult # create alias -- see #17805 # # Class whose instances are returned by `Pool.map_async()` # class MapResult(ApplyResult): def __init__(self, cache, chunksize, length, callback, error_callback): ApplyResult.__init__(self, cache, callback, error_callback=error_callback) self._success = True self._value = [None] * length self._chunksize = chunksize if chunksize <= 0: self._number_left = 0 self._event.set() del cache[self._job] else: self._number_left = length//chunksize + bool(length % chunksize) def _set(self, i, success_result): self._number_left -= 1 success, result = success_result if success and self._success: self._value[i*self._chunksize:(i+1)*self._chunksize] = result if self._number_left == 0: if self._callback: self._callback(self._value) del self._cache[self._job] self._event.set() else: if not success and self._success: # only store first exception self._success = False self._value = result if self._number_left == 0: # only consider the result ready once all jobs are done if self._error_callback: self._error_callback(self._value) del self._cache[self._job] self._event.set() # # Class whose instances are returned by `Pool.imap()` # class IMapIterator(object): def __init__(self, cache): self._cond = threading.Condition(threading.Lock()) self._job = next(job_counter) self._cache = cache self._items = collections.deque() self._index = 0 self._length = None self._unsorted = {} cache[self._job] = self def __iter__(self): return self def next(self, timeout=None): with self._cond: try: item = self._items.popleft() except IndexError: if self._index == self._length: raise StopIteration self._cond.wait(timeout) try: item = self._items.popleft() except IndexError: if self._index == self._length: raise StopIteration raise TimeoutError success, value = item if success: return value raise value __next__ = next # XXX def _set(self, i, obj): with self._cond: if self._index == i: self._items.append(obj) self._index += 1 while self._index in self._unsorted: obj = self._unsorted.pop(self._index) self._items.append(obj) self._index += 1 self._cond.notify() else: self._unsorted[i] = obj if self._index == self._length: del self._cache[self._job] def _set_length(self, length): with self._cond: self._length = length if self._index == self._length: self._cond.notify() del self._cache[self._job] # # Class whose instances are returned by `Pool.imap_unordered()` # class IMapUnorderedIterator(IMapIterator): def _set(self, i, obj): with self._cond: self._items.append(obj) self._index += 1 self._cond.notify() if self._index == self._length: del self._cache[self._job] # # # class ThreadPool(Pool): _wrap_exception = False @staticmethod def Process(*args, **kwds): from .dummy import Process return Process(*args, **kwds) def __init__(self, processes=None, initializer=None, initargs=()): Pool.__init__(self, processes, initializer, initargs) def _setup_queues(self): self._inqueue = queue.Queue() self._outqueue = queue.Queue() self._quick_put = self._inqueue.put self._quick_get = self._outqueue.get @staticmethod def _help_stuff_finish(inqueue, task_handler, size): # put sentinels at head of inqueue to make workers finish with inqueue.not_empty: inqueue.queue.clear() inqueue.queue.extend([None] * size) inqueue.not_empty.notify_all() popen_fork.py 0000644 00000004403 15204243001 0007250 0 ustar 00 import os import sys import signal from . import util __all__ = ['Popen'] # # Start child process using fork # class Popen(object): method = 'fork' def __init__(self, process_obj): util._flush_std_streams() self.returncode = None self._launch(process_obj) def duplicate_for_child(self, fd): return fd def poll(self, flag=os.WNOHANG): if self.returncode is None: while True: try: pid, sts = os.waitpid(self.pid, flag) except OSError as e: # Child process not yet created. See #1731717 # e.errno == errno.ECHILD == 10 return None else: break if pid == self.pid: if os.WIFSIGNALED(sts): self.returncode = -os.WTERMSIG(sts) else: assert os.WIFEXITED(sts) self.returncode = os.WEXITSTATUS(sts) return self.returncode def wait(self, timeout=None): if self.returncode is None: if timeout is not None: from multiprocessing.connection import wait if not wait([self.sentinel], timeout): return None # This shouldn't block if wait() returned successfully. return self.poll(os.WNOHANG if timeout == 0.0 else 0) return self.returncode def terminate(self): if self.returncode is None: try: os.kill(self.pid, signal.SIGTERM) except ProcessLookupError: pass except OSError: if self.wait(timeout=0.1) is None: raise def _launch(self, process_obj): code = 1 parent_r, child_w = os.pipe() self.pid = os.fork() if self.pid == 0: try: os.close(parent_r) if 'random' in sys.modules: import random random.seed() code = process_obj._bootstrap() finally: os._exit(code) else: os.close(child_w) util.Finalize(self, os.close, (parent_r,)) self.sentinel = parent_r connection.py 0000644 00000074526 15204243001 0007262 0 ustar 00 # # A higher level module for using sockets (or Windows named pipes) # # multiprocessing/connection.py # # Copyright (c) 2006-2008, R Oudkerk # Licensed to PSF under a Contributor Agreement. # __all__ = [ 'Client', 'Listener', 'Pipe', 'wait' ] import io import os import sys import socket import struct import time import tempfile import itertools import _multiprocessing from . import util from . import AuthenticationError, BufferTooShort from .context import reduction _ForkingPickler = reduction.ForkingPickler try: import _winapi from _winapi import WAIT_OBJECT_0, WAIT_ABANDONED_0, WAIT_TIMEOUT, INFINITE except ImportError: if sys.platform == 'win32': raise _winapi = None # # # BUFSIZE = 8192 # A very generous timeout when it comes to local connections... CONNECTION_TIMEOUT = 20. # The hmac module implicitly defaults to using MD5. # Support using a stronger algorithm for the challenge/response code: HMAC_DIGEST_NAME='sha256' _mmap_counter = itertools.count() default_family = 'AF_INET' families = ['AF_INET'] if hasattr(socket, 'AF_UNIX'): default_family = 'AF_UNIX' families += ['AF_UNIX'] if sys.platform == 'win32': default_family = 'AF_PIPE' families += ['AF_PIPE'] def _init_timeout(timeout=CONNECTION_TIMEOUT): return time.monotonic() + timeout def _check_timeout(t): return time.monotonic() > t # # # def arbitrary_address(family): ''' Return an arbitrary free address for the given family ''' if family == 'AF_INET': return ('localhost', 0) elif family == 'AF_UNIX': return tempfile.mktemp(prefix='listener-', dir=util.get_temp_dir()) elif family == 'AF_PIPE': return tempfile.mktemp(prefix=r'\\.\pipe\pyc-%d-%d-' % (os.getpid(), next(_mmap_counter)), dir="") else: raise ValueError('unrecognized family') def _validate_family(family): ''' Checks if the family is valid for the current environment. ''' if sys.platform != 'win32' and family == 'AF_PIPE': raise ValueError('Family %s is not recognized.' % family) if sys.platform == 'win32' and family == 'AF_UNIX': # double check if not hasattr(socket, family): raise ValueError('Family %s is not recognized.' % family) def address_type(address): ''' Return the types of the address This can be 'AF_INET', 'AF_UNIX', or 'AF_PIPE' ''' if type(address) == tuple: return 'AF_INET' elif type(address) is str and address.startswith('\\\\'): return 'AF_PIPE' elif type(address) is str: return 'AF_UNIX' else: raise ValueError('address type of %r unrecognized' % address) # # Connection classes # class _ConnectionBase: _handle = None def __init__(self, handle, readable=True, writable=True): handle = handle.__index__() if handle < 0: raise ValueError("invalid handle") if not readable and not writable: raise ValueError( "at least one of `readable` and `writable` must be True") self._handle = handle self._readable = readable self._writable = writable # XXX should we use util.Finalize instead of a __del__? def __del__(self): if self._handle is not None: self._close() def _check_closed(self): if self._handle is None: raise OSError("handle is closed") def _check_readable(self): if not self._readable: raise OSError("connection is write-only") def _check_writable(self): if not self._writable: raise OSError("connection is read-only") def _bad_message_length(self): if self._writable: self._readable = False else: self.close() raise OSError("bad message length") @property def closed(self): """True if the connection is closed""" return self._handle is None @property def readable(self): """True if the connection is readable""" return self._readable @property def writable(self): """True if the connection is writable""" return self._writable def fileno(self): """File descriptor or handle of the connection""" self._check_closed() return self._handle def close(self): """Close the connection""" if self._handle is not None: try: self._close() finally: self._handle = None def send_bytes(self, buf, offset=0, size=None): """Send the bytes data from a bytes-like object""" self._check_closed() self._check_writable() m = memoryview(buf) # HACK for byte-indexing of non-bytewise buffers (e.g. array.array) if m.itemsize > 1: m = memoryview(bytes(m)) n = len(m) if offset < 0: raise ValueError("offset is negative") if n < offset: raise ValueError("buffer length < offset") if size is None: size = n - offset elif size < 0: raise ValueError("size is negative") elif offset + size > n: raise ValueError("buffer length < offset + size") self._send_bytes(m[offset:offset + size]) def send(self, obj): """Send a (picklable) object""" self._check_closed() self._check_writable() self._send_bytes(_ForkingPickler.dumps(obj)) def recv_bytes(self, maxlength=None): """ Receive bytes data as a bytes object. """ self._check_closed() self._check_readable() if maxlength is not None and maxlength < 0: raise ValueError("negative maxlength") buf = self._recv_bytes(maxlength) if buf is None: self._bad_message_length() return buf.getvalue() def recv_bytes_into(self, buf, offset=0): """ Receive bytes data into a writeable bytes-like object. Return the number of bytes read. """ self._check_closed() self._check_readable() with memoryview(buf) as m: # Get bytesize of arbitrary buffer itemsize = m.itemsize bytesize = itemsize * len(m) if offset < 0: raise ValueError("negative offset") elif offset > bytesize: raise ValueError("offset too large") result = self._recv_bytes() size = result.tell() if bytesize < offset + size: raise BufferTooShort(result.getvalue()) # Message can fit in dest result.seek(0) result.readinto(m[offset // itemsize : (offset + size) // itemsize]) return size def recv(self): """Receive a (picklable) object""" self._check_closed() self._check_readable() buf = self._recv_bytes() return _ForkingPickler.loads(buf.getbuffer()) def poll(self, timeout=0.0): """Whether there is any input available to be read""" self._check_closed() self._check_readable() return self._poll(timeout) def __enter__(self): return self def __exit__(self, exc_type, exc_value, exc_tb): self.close() if _winapi: class PipeConnection(_ConnectionBase): """ Connection class based on a Windows named pipe. Overlapped I/O is used, so the handles must have been created with FILE_FLAG_OVERLAPPED. """ _got_empty_message = False def _close(self, _CloseHandle=_winapi.CloseHandle): _CloseHandle(self._handle) def _send_bytes(self, buf): ov, err = _winapi.WriteFile(self._handle, buf, overlapped=True) try: if err == _winapi.ERROR_IO_PENDING: waitres = _winapi.WaitForMultipleObjects( [ov.event], False, INFINITE) assert waitres == WAIT_OBJECT_0 except: ov.cancel() raise finally: nwritten, err = ov.GetOverlappedResult(True) assert err == 0 assert nwritten == len(buf) def _recv_bytes(self, maxsize=None): if self._got_empty_message: self._got_empty_message = False return io.BytesIO() else: bsize = 128 if maxsize is None else min(maxsize, 128) try: ov, err = _winapi.ReadFile(self._handle, bsize, overlapped=True) try: if err == _winapi.ERROR_IO_PENDING: waitres = _winapi.WaitForMultipleObjects( [ov.event], False, INFINITE) assert waitres == WAIT_OBJECT_0 except: ov.cancel() raise finally: nread, err = ov.GetOverlappedResult(True) if err == 0: f = io.BytesIO() f.write(ov.getbuffer()) return f elif err == _winapi.ERROR_MORE_DATA: return self._get_more_data(ov, maxsize) except OSError as e: if e.winerror == _winapi.ERROR_BROKEN_PIPE: raise EOFError else: raise raise RuntimeError("shouldn't get here; expected KeyboardInterrupt") def _poll(self, timeout): if (self._got_empty_message or _winapi.PeekNamedPipe(self._handle)[0] != 0): return True return bool(wait([self], timeout)) def _get_more_data(self, ov, maxsize): buf = ov.getbuffer() f = io.BytesIO() f.write(buf) left = _winapi.PeekNamedPipe(self._handle)[1] assert left > 0 if maxsize is not None and len(buf) + left > maxsize: self._bad_message_length() ov, err = _winapi.ReadFile(self._handle, left, overlapped=True) rbytes, err = ov.GetOverlappedResult(True) assert err == 0 assert rbytes == left f.write(ov.getbuffer()) return f class Connection(_ConnectionBase): """ Connection class based on an arbitrary file descriptor (Unix only), or a socket handle (Windows). """ if _winapi: def _close(self, _close=_multiprocessing.closesocket): _close(self._handle) _write = _multiprocessing.send _read = _multiprocessing.recv else: def _close(self, _close=os.close): _close(self._handle) _write = os.write _read = os.read def _send(self, buf, write=_write): remaining = len(buf) while True: n = write(self._handle, buf) remaining -= n if remaining == 0: break buf = buf[n:] def _recv(self, size, read=_read): buf = io.BytesIO() handle = self._handle remaining = size while remaining > 0: chunk = read(handle, remaining) n = len(chunk) if n == 0: if remaining == size: raise EOFError else: raise OSError("got end of file during message") buf.write(chunk) remaining -= n return buf def _send_bytes(self, buf): n = len(buf) # For wire compatibility with 3.2 and lower header = struct.pack("!i", n) if n > 16384: # The payload is large so Nagle's algorithm won't be triggered # and we'd better avoid the cost of concatenation. self._send(header) self._send(buf) else: # Issue #20540: concatenate before sending, to avoid delays due # to Nagle's algorithm on a TCP socket. # Also note we want to avoid sending a 0-length buffer separately, # to avoid "broken pipe" errors if the other end closed the pipe. self._send(header + buf) def _recv_bytes(self, maxsize=None): buf = self._recv(4) size, = struct.unpack("!i", buf.getvalue()) if maxsize is not None and size > maxsize: return None return self._recv(size) def _poll(self, timeout): r = wait([self], timeout) return bool(r) # # Public functions # class Listener(object): ''' Returns a listener object. This is a wrapper for a bound socket which is 'listening' for connections, or for a Windows named pipe. ''' def __init__(self, address=None, family=None, backlog=1, authkey=None): family = family or (address and address_type(address)) \ or default_family address = address or arbitrary_address(family) _validate_family(family) if family == 'AF_PIPE': self._listener = PipeListener(address, backlog) else: self._listener = SocketListener(address, family, backlog) if authkey is not None and not isinstance(authkey, bytes): raise TypeError('authkey should be a byte string') self._authkey = authkey def accept(self): ''' Accept a connection on the bound socket or named pipe of `self`. Returns a `Connection` object. ''' if self._listener is None: raise OSError('listener is closed') c = self._listener.accept() if self._authkey: deliver_challenge(c, self._authkey) answer_challenge(c, self._authkey) return c def close(self): ''' Close the bound socket or named pipe of `self`. ''' listener = self._listener if listener is not None: self._listener = None listener.close() address = property(lambda self: self._listener._address) last_accepted = property(lambda self: self._listener._last_accepted) def __enter__(self): return self def __exit__(self, exc_type, exc_value, exc_tb): self.close() def Client(address, family=None, authkey=None): ''' Returns a connection to the address of a `Listener` ''' family = family or address_type(address) _validate_family(family) if family == 'AF_PIPE': c = PipeClient(address) else: c = SocketClient(address) if authkey is not None and not isinstance(authkey, bytes): raise TypeError('authkey should be a byte string') if authkey is not None: answer_challenge(c, authkey) deliver_challenge(c, authkey) return c if sys.platform != 'win32': def Pipe(duplex=True): ''' Returns pair of connection objects at either end of a pipe ''' if duplex: s1, s2 = socket.socketpair() s1.setblocking(True) s2.setblocking(True) c1 = Connection(s1.detach()) c2 = Connection(s2.detach()) else: fd1, fd2 = os.pipe() c1 = Connection(fd1, writable=False) c2 = Connection(fd2, readable=False) return c1, c2 else: def Pipe(duplex=True): ''' Returns pair of connection objects at either end of a pipe ''' address = arbitrary_address('AF_PIPE') if duplex: openmode = _winapi.PIPE_ACCESS_DUPLEX access = _winapi.GENERIC_READ | _winapi.GENERIC_WRITE obsize, ibsize = BUFSIZE, BUFSIZE else: openmode = _winapi.PIPE_ACCESS_INBOUND access = _winapi.GENERIC_WRITE obsize, ibsize = 0, BUFSIZE h1 = _winapi.CreateNamedPipe( address, openmode | _winapi.FILE_FLAG_OVERLAPPED | _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE, _winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE | _winapi.PIPE_WAIT, 1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER, # default security descriptor: the handle cannot be inherited _winapi.NULL ) h2 = _winapi.CreateFile( address, access, 0, _winapi.NULL, _winapi.OPEN_EXISTING, _winapi.FILE_FLAG_OVERLAPPED, _winapi.NULL ) _winapi.SetNamedPipeHandleState( h2, _winapi.PIPE_READMODE_MESSAGE, None, None ) overlapped = _winapi.ConnectNamedPipe(h1, overlapped=True) _, err = overlapped.GetOverlappedResult(True) assert err == 0 c1 = PipeConnection(h1, writable=duplex) c2 = PipeConnection(h2, readable=duplex) return c1, c2 # # Definitions for connections based on sockets # class SocketListener(object): ''' Representation of a socket which is bound to an address and listening ''' def __init__(self, address, family, backlog=1): self._socket = socket.socket(getattr(socket, family)) try: # SO_REUSEADDR has different semantics on Windows (issue #2550). if os.name == 'posix': self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self._socket.setblocking(True) self._socket.bind(address) self._socket.listen(backlog) self._address = self._socket.getsockname() except OSError: self._socket.close() raise self._family = family self._last_accepted = None if family == 'AF_UNIX': self._unlink = util.Finalize( self, os.unlink, args=(address,), exitpriority=0 ) else: self._unlink = None def accept(self): s, self._last_accepted = self._socket.accept() s.setblocking(True) return Connection(s.detach()) def close(self): try: self._socket.close() finally: unlink = self._unlink if unlink is not None: self._unlink = None unlink() def SocketClient(address): ''' Return a connection object connected to the socket given by `address` ''' family = address_type(address) with socket.socket( getattr(socket, family) ) as s: s.setblocking(True) s.connect(address) return Connection(s.detach()) # # Definitions for connections based on named pipes # if sys.platform == 'win32': class PipeListener(object): ''' Representation of a named pipe ''' def __init__(self, address, backlog=None): self._address = address self._handle_queue = [self._new_handle(first=True)] self._last_accepted = None util.sub_debug('listener created with address=%r', self._address) self.close = util.Finalize( self, PipeListener._finalize_pipe_listener, args=(self._handle_queue, self._address), exitpriority=0 ) def _new_handle(self, first=False): flags = _winapi.PIPE_ACCESS_DUPLEX | _winapi.FILE_FLAG_OVERLAPPED if first: flags |= _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE return _winapi.CreateNamedPipe( self._address, flags, _winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE | _winapi.PIPE_WAIT, _winapi.PIPE_UNLIMITED_INSTANCES, BUFSIZE, BUFSIZE, _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL ) def accept(self): self._handle_queue.append(self._new_handle()) handle = self._handle_queue.pop(0) try: ov = _winapi.ConnectNamedPipe(handle, overlapped=True) except OSError as e: if e.winerror != _winapi.ERROR_NO_DATA: raise # ERROR_NO_DATA can occur if a client has already connected, # written data and then disconnected -- see Issue 14725. else: try: res = _winapi.WaitForMultipleObjects( [ov.event], False, INFINITE) except: ov.cancel() _winapi.CloseHandle(handle) raise finally: _, err = ov.GetOverlappedResult(True) assert err == 0 return PipeConnection(handle) @staticmethod def _finalize_pipe_listener(queue, address): util.sub_debug('closing listener with address=%r', address) for handle in queue: _winapi.CloseHandle(handle) def PipeClient(address): ''' Return a connection object connected to the pipe given by `address` ''' t = _init_timeout() while 1: try: _winapi.WaitNamedPipe(address, 1000) h = _winapi.CreateFile( address, _winapi.GENERIC_READ | _winapi.GENERIC_WRITE, 0, _winapi.NULL, _winapi.OPEN_EXISTING, _winapi.FILE_FLAG_OVERLAPPED, _winapi.NULL ) except OSError as e: if e.winerror not in (_winapi.ERROR_SEM_TIMEOUT, _winapi.ERROR_PIPE_BUSY) or _check_timeout(t): raise else: break else: raise _winapi.SetNamedPipeHandleState( h, _winapi.PIPE_READMODE_MESSAGE, None, None ) return PipeConnection(h) # # Authentication stuff # MESSAGE_LENGTH = 20 CHALLENGE = b'#CHALLENGE#' WELCOME = b'#WELCOME#' FAILURE = b'#FAILURE#' def deliver_challenge(connection, authkey): import hmac assert isinstance(authkey, bytes) message = os.urandom(MESSAGE_LENGTH) connection.send_bytes(CHALLENGE + message) digest = hmac.new(authkey, message, HMAC_DIGEST_NAME).digest() response = connection.recv_bytes(256) # reject large message if response == digest: connection.send_bytes(WELCOME) else: connection.send_bytes(FAILURE) raise AuthenticationError('digest received was wrong') def answer_challenge(connection, authkey): import hmac assert isinstance(authkey, bytes) message = connection.recv_bytes(256) # reject large message assert message[:len(CHALLENGE)] == CHALLENGE, 'message = %r' % message message = message[len(CHALLENGE):] digest = hmac.new(authkey, message, HMAC_DIGEST_NAME).digest() connection.send_bytes(digest) response = connection.recv_bytes(256) # reject large message if response != WELCOME: raise AuthenticationError('digest sent was rejected') # # Support for using xmlrpclib for serialization # class ConnectionWrapper(object): def __init__(self, conn, dumps, loads): self._conn = conn self._dumps = dumps self._loads = loads for attr in ('fileno', 'close', 'poll', 'recv_bytes', 'send_bytes'): obj = getattr(conn, attr) setattr(self, attr, obj) def send(self, obj): s = self._dumps(obj) self._conn.send_bytes(s) def recv(self): s = self._conn.recv_bytes() return self._loads(s) def _xml_dumps(obj): return xmlrpclib.dumps((obj,), None, None, None, 1).encode('utf-8') def _xml_loads(s): (obj,), method = xmlrpclib.loads(s.decode('utf-8')) return obj class XmlListener(Listener): def accept(self): global xmlrpclib import xmlrpc.client as xmlrpclib obj = Listener.accept(self) return ConnectionWrapper(obj, _xml_dumps, _xml_loads) def XmlClient(*args, **kwds): global xmlrpclib import xmlrpc.client as xmlrpclib return ConnectionWrapper(Client(*args, **kwds), _xml_dumps, _xml_loads) # # Wait # if sys.platform == 'win32': def _exhaustive_wait(handles, timeout): # Return ALL handles which are currently signalled. (Only # returning the first signalled might create starvation issues.) L = list(handles) ready = [] while L: res = _winapi.WaitForMultipleObjects(L, False, timeout) if res == WAIT_TIMEOUT: break elif WAIT_OBJECT_0 <= res < WAIT_OBJECT_0 + len(L): res -= WAIT_OBJECT_0 elif WAIT_ABANDONED_0 <= res < WAIT_ABANDONED_0 + len(L): res -= WAIT_ABANDONED_0 else: raise RuntimeError('Should not get here') ready.append(L[res]) L = L[res+1:] timeout = 0 return ready _ready_errors = {_winapi.ERROR_BROKEN_PIPE, _winapi.ERROR_NETNAME_DELETED} def wait(object_list, timeout=None): ''' Wait till an object in object_list is ready/readable. Returns list of those objects in object_list which are ready/readable. ''' if timeout is None: timeout = INFINITE elif timeout < 0: timeout = 0 else: timeout = int(timeout * 1000 + 0.5) object_list = list(object_list) waithandle_to_obj = {} ov_list = [] ready_objects = set() ready_handles = set() try: for o in object_list: try: fileno = getattr(o, 'fileno') except AttributeError: waithandle_to_obj[o.__index__()] = o else: # start an overlapped read of length zero try: ov, err = _winapi.ReadFile(fileno(), 0, True) except OSError as e: ov, err = None, e.winerror if err not in _ready_errors: raise if err == _winapi.ERROR_IO_PENDING: ov_list.append(ov) waithandle_to_obj[ov.event] = o else: # If o.fileno() is an overlapped pipe handle and # err == 0 then there is a zero length message # in the pipe, but it HAS NOT been consumed... if ov and sys.getwindowsversion()[:2] >= (6, 2): # ... except on Windows 8 and later, where # the message HAS been consumed. try: _, err = ov.GetOverlappedResult(False) except OSError as e: err = e.winerror if not err and hasattr(o, '_got_empty_message'): o._got_empty_message = True ready_objects.add(o) timeout = 0 ready_handles = _exhaustive_wait(waithandle_to_obj.keys(), timeout) finally: # request that overlapped reads stop for ov in ov_list: ov.cancel() # wait for all overlapped reads to stop for ov in ov_list: try: _, err = ov.GetOverlappedResult(True) except OSError as e: err = e.winerror if err not in _ready_errors: raise if err != _winapi.ERROR_OPERATION_ABORTED: o = waithandle_to_obj[ov.event] ready_objects.add(o) if err == 0: # If o.fileno() is an overlapped pipe handle then # a zero length message HAS been consumed. if hasattr(o, '_got_empty_message'): o._got_empty_message = True ready_objects.update(waithandle_to_obj[h] for h in ready_handles) return [o for o in object_list if o in ready_objects] else: import selectors # poll/select have the advantage of not requiring any extra file # descriptor, contrarily to epoll/kqueue (also, they require a single # syscall). if hasattr(selectors, 'PollSelector'): _WaitSelector = selectors.PollSelector else: _WaitSelector = selectors.SelectSelector def wait(object_list, timeout=None): ''' Wait till an object in object_list is ready/readable. Returns list of those objects in object_list which are ready/readable. ''' with _WaitSelector() as selector: for obj in object_list: selector.register(obj, selectors.EVENT_READ) if timeout is not None: deadline = time.monotonic() + timeout while True: ready = selector.select(timeout) if ready: return [key.fileobj for (key, events) in ready] else: if timeout is not None: timeout = deadline - time.monotonic() if timeout < 0: return ready # # Make connection and socket objects sharable if possible # if sys.platform == 'win32': def reduce_connection(conn): handle = conn.fileno() with socket.fromfd(handle, socket.AF_INET, socket.SOCK_STREAM) as s: from . import resource_sharer ds = resource_sharer.DupSocket(s) return rebuild_connection, (ds, conn.readable, conn.writable) def rebuild_connection(ds, readable, writable): sock = ds.detach() return Connection(sock.detach(), readable, writable) reduction.register(Connection, reduce_connection) def reduce_pipe_connection(conn): access = ((_winapi.FILE_GENERIC_READ if conn.readable else 0) | (_winapi.FILE_GENERIC_WRITE if conn.writable else 0)) dh = reduction.DupHandle(conn.fileno(), access) return rebuild_pipe_connection, (dh, conn.readable, conn.writable) def rebuild_pipe_connection(dh, readable, writable): handle = dh.detach() return PipeConnection(handle, readable, writable) reduction.register(PipeConnection, reduce_pipe_connection) else: def reduce_connection(conn): df = reduction.DupFd(conn.fileno()) return rebuild_connection, (df, conn.readable, conn.writable) def rebuild_connection(df, readable, writable): fd = df.detach() return Connection(fd, readable, writable) reduction.register(Connection, reduce_connection) util.py 0000644 00000027156 15204243002 0006076 0 ustar 00 # # Module providing various facilities to other parts of the package # # multiprocessing/util.py # # Copyright (c) 2006-2008, R Oudkerk # Licensed to PSF under a Contributor Agreement. # import os import itertools import sys import weakref import atexit import threading # we want threading to install it's # cleanup function before multiprocessing does from subprocess import _args_from_interpreter_flags from . import process __all__ = [ 'sub_debug', 'debug', 'info', 'sub_warning', 'get_logger', 'log_to_stderr', 'get_temp_dir', 'register_after_fork', 'is_exiting', 'Finalize', 'ForkAwareThreadLock', 'ForkAwareLocal', 'close_all_fds_except', 'SUBDEBUG', 'SUBWARNING', ] # # Logging # NOTSET = 0 SUBDEBUG = 5 DEBUG = 10 INFO = 20 SUBWARNING = 25 LOGGER_NAME = 'multiprocessing' DEFAULT_LOGGING_FORMAT = '[%(levelname)s/%(processName)s] %(message)s' _logger = None _log_to_stderr = False def sub_debug(msg, *args): if _logger: _logger.log(SUBDEBUG, msg, *args) def debug(msg, *args): if _logger: _logger.log(DEBUG, msg, *args) def info(msg, *args): if _logger: _logger.log(INFO, msg, *args) def sub_warning(msg, *args): if _logger: _logger.log(SUBWARNING, msg, *args) def get_logger(): ''' Returns logger used by multiprocessing ''' global _logger import logging logging._acquireLock() try: if not _logger: _logger = logging.getLogger(LOGGER_NAME) _logger.propagate = 0 # XXX multiprocessing should cleanup before logging if hasattr(atexit, 'unregister'): atexit.unregister(_exit_function) atexit.register(_exit_function) else: atexit._exithandlers.remove((_exit_function, (), {})) atexit._exithandlers.append((_exit_function, (), {})) finally: logging._releaseLock() return _logger def log_to_stderr(level=None): ''' Turn on logging and add a handler which prints to stderr ''' global _log_to_stderr import logging logger = get_logger() formatter = logging.Formatter(DEFAULT_LOGGING_FORMAT) handler = logging.StreamHandler() handler.setFormatter(formatter) logger.addHandler(handler) if level: logger.setLevel(level) _log_to_stderr = True return _logger # # Function returning a temp directory which will be removed on exit # def get_temp_dir(): # get name of a temp directory which will be automatically cleaned up tempdir = process.current_process()._config.get('tempdir') if tempdir is None: import shutil, tempfile tempdir = tempfile.mkdtemp(prefix='pymp-') info('created temp directory %s', tempdir) Finalize(None, shutil.rmtree, args=[tempdir], exitpriority=-100) process.current_process()._config['tempdir'] = tempdir return tempdir # # Support for reinitialization of objects when bootstrapping a child process # _afterfork_registry = weakref.WeakValueDictionary() _afterfork_counter = itertools.count() def _run_after_forkers(): items = list(_afterfork_registry.items()) items.sort() for (index, ident, func), obj in items: try: func(obj) except Exception as e: info('after forker raised exception %s', e) def register_after_fork(obj, func): _afterfork_registry[(next(_afterfork_counter), id(obj), func)] = obj # # Finalization using weakrefs # _finalizer_registry = {} _finalizer_counter = itertools.count() class Finalize(object): ''' Class which supports object finalization using weakrefs ''' def __init__(self, obj, callback, args=(), kwargs=None, exitpriority=None): assert exitpriority is None or type(exitpriority) is int if obj is not None: self._weakref = weakref.ref(obj, self) else: assert exitpriority is not None self._callback = callback self._args = args self._kwargs = kwargs or {} self._key = (exitpriority, next(_finalizer_counter)) self._pid = os.getpid() _finalizer_registry[self._key] = self def __call__(self, wr=None, # Need to bind these locally because the globals can have # been cleared at shutdown _finalizer_registry=_finalizer_registry, sub_debug=sub_debug, getpid=os.getpid): ''' Run the callback unless it has already been called or cancelled ''' try: del _finalizer_registry[self._key] except KeyError: sub_debug('finalizer no longer registered') else: if self._pid != getpid(): sub_debug('finalizer ignored because different process') res = None else: sub_debug('finalizer calling %s with args %s and kwargs %s', self._callback, self._args, self._kwargs) res = self._callback(*self._args, **self._kwargs) self._weakref = self._callback = self._args = \ self._kwargs = self._key = None return res def cancel(self): ''' Cancel finalization of the object ''' try: del _finalizer_registry[self._key] except KeyError: pass else: self._weakref = self._callback = self._args = \ self._kwargs = self._key = None def still_active(self): ''' Return whether this finalizer is still waiting to invoke callback ''' return self._key in _finalizer_registry def __repr__(self): try: obj = self._weakref() except (AttributeError, TypeError): obj = None if obj is None: return '<%s object, dead>' % self.__class__.__name__ x = '<%s object, callback=%s' % ( self.__class__.__name__, getattr(self._callback, '__name__', self._callback)) if self._args: x += ', args=' + str(self._args) if self._kwargs: x += ', kwargs=' + str(self._kwargs) if self._key[0] is not None: x += ', exitprority=' + str(self._key[0]) return x + '>' def _run_finalizers(minpriority=None): ''' Run all finalizers whose exit priority is not None and at least minpriority Finalizers with highest priority are called first; finalizers with the same priority will be called in reverse order of creation. ''' if _finalizer_registry is None: # This function may be called after this module's globals are # destroyed. See the _exit_function function in this module for more # notes. return if minpriority is None: f = lambda p : p[0] is not None else: f = lambda p : p[0] is not None and p[0] >= minpriority # Careful: _finalizer_registry may be mutated while this function # is running (either by a GC run or by another thread). # list(_finalizer_registry) should be atomic, while # list(_finalizer_registry.items()) is not. keys = [key for key in list(_finalizer_registry) if f(key)] keys.sort(reverse=True) for key in keys: finalizer = _finalizer_registry.get(key) # key may have been removed from the registry if finalizer is not None: sub_debug('calling %s', finalizer) try: finalizer() except Exception: import traceback traceback.print_exc() if minpriority is None: _finalizer_registry.clear() # # Clean up on exit # def is_exiting(): ''' Returns true if the process is shutting down ''' return _exiting or _exiting is None _exiting = False def _exit_function(info=info, debug=debug, _run_finalizers=_run_finalizers, active_children=process.active_children, current_process=process.current_process): # We hold on to references to functions in the arglist due to the # situation described below, where this function is called after this # module's globals are destroyed. global _exiting if not _exiting: _exiting = True info('process shutting down') debug('running all "atexit" finalizers with priority >= 0') _run_finalizers(0) if current_process() is not None: # We check if the current process is None here because if # it's None, any call to ``active_children()`` will raise # an AttributeError (active_children winds up trying to # get attributes from util._current_process). One # situation where this can happen is if someone has # manipulated sys.modules, causing this module to be # garbage collected. The destructor for the module type # then replaces all values in the module dict with None. # For instance, after setuptools runs a test it replaces # sys.modules with a copy created earlier. See issues # #9775 and #15881. Also related: #4106, #9205, and # #9207. for p in active_children(): if p.daemon: info('calling terminate() for daemon %s', p.name) p._popen.terminate() for p in active_children(): info('calling join() for process %s', p.name) p.join() debug('running the remaining "atexit" finalizers') _run_finalizers() atexit.register(_exit_function) # # Some fork aware types # class ForkAwareThreadLock(object): def __init__(self): self._reset() register_after_fork(self, ForkAwareThreadLock._reset) def _reset(self): self._lock = threading.Lock() self.acquire = self._lock.acquire self.release = self._lock.release def __enter__(self): return self._lock.__enter__() def __exit__(self, *args): return self._lock.__exit__(*args) class ForkAwareLocal(threading.local): def __init__(self): register_after_fork(self, lambda obj : obj.__dict__.clear()) def __reduce__(self): return type(self), () # # Close fds except those specified # try: MAXFD = os.sysconf("SC_OPEN_MAX") except Exception: MAXFD = 256 def close_all_fds_except(fds): fds = list(fds) + [-1, MAXFD] fds.sort() assert fds[-1] == MAXFD, 'fd too large' for i in range(len(fds) - 1): os.closerange(fds[i]+1, fds[i+1]) # # Close sys.stdin and replace stdin with os.devnull # def _close_stdin(): if sys.stdin is None: return try: sys.stdin.close() except (OSError, ValueError): pass try: fd = os.open(os.devnull, os.O_RDONLY) try: sys.stdin = open(fd, closefd=False) except: os.close(fd) raise except (OSError, ValueError): pass # # Flush standard streams, if any # def _flush_std_streams(): try: sys.stdout.flush() except (AttributeError, ValueError): pass try: sys.stderr.flush() except (AttributeError, ValueError): pass # # Start a program with only specified fds kept open # def spawnv_passfds(path, args, passfds): import _posixsubprocess passfds = tuple(sorted(map(int, passfds))) errpipe_read, errpipe_write = os.pipe() try: return _posixsubprocess.fork_exec( args, [os.fsencode(path)], True, passfds, None, None, -1, -1, -1, -1, -1, -1, errpipe_read, errpipe_write, False, False, None) finally: os.close(errpipe_read) os.close(errpipe_write) heap.py 0000644 00000020177 15204243002 0006032 0 ustar 00 # # Module which supports allocation of memory from an mmap # # multiprocessing/heap.py # # Copyright (c) 2006-2008, R Oudkerk # Licensed to PSF under a Contributor Agreement. # import bisect import mmap import os import sys import tempfile import threading from .context import reduction, assert_spawning from . import util __all__ = ['BufferWrapper'] # # Inheritable class which wraps an mmap, and from which blocks can be allocated # if sys.platform == 'win32': import _winapi class Arena(object): _rand = tempfile._RandomNameSequence() def __init__(self, size): self.size = size for i in range(100): name = 'pym-%d-%s' % (os.getpid(), next(self._rand)) buf = mmap.mmap(-1, size, tagname=name) if _winapi.GetLastError() == 0: break # We have reopened a preexisting mmap. buf.close() else: raise FileExistsError('Cannot find name for new mmap') self.name = name self.buffer = buf self._state = (self.size, self.name) def __getstate__(self): assert_spawning(self) return self._state def __setstate__(self, state): self.size, self.name = self._state = state self.buffer = mmap.mmap(-1, self.size, tagname=self.name) # XXX Temporarily preventing buildbot failures while determining # XXX the correct long-term fix. See issue 23060 #assert _winapi.GetLastError() == _winapi.ERROR_ALREADY_EXISTS else: class Arena(object): def __init__(self, size, fd=-1): self.size = size self.fd = fd if fd == -1: self.fd, name = tempfile.mkstemp( prefix='pym-%d-'%os.getpid(), dir=util.get_temp_dir()) os.unlink(name) util.Finalize(self, os.close, (self.fd,)) with open(self.fd, 'wb', closefd=False) as f: bs = 1024 * 1024 if size >= bs: zeros = b'\0' * bs for _ in range(size // bs): f.write(zeros) del zeros f.write(b'\0' * (size % bs)) assert f.tell() == size self.buffer = mmap.mmap(self.fd, self.size) def reduce_arena(a): if a.fd == -1: raise ValueError('Arena is unpicklable because ' 'forking was enabled when it was created') return rebuild_arena, (a.size, reduction.DupFd(a.fd)) def rebuild_arena(size, dupfd): return Arena(size, dupfd.detach()) reduction.register(Arena, reduce_arena) # # Class allowing allocation of chunks of memory from arenas # class Heap(object): _alignment = 8 def __init__(self, size=mmap.PAGESIZE): self._lastpid = os.getpid() self._lock = threading.Lock() self._size = size self._lengths = [] self._len_to_seq = {} self._start_to_block = {} self._stop_to_block = {} self._allocated_blocks = set() self._arenas = [] # list of pending blocks to free - see free() comment below self._pending_free_blocks = [] @staticmethod def _roundup(n, alignment): # alignment must be a power of 2 mask = alignment - 1 return (n + mask) & ~mask def _malloc(self, size): # returns a large enough block -- it might be much larger i = bisect.bisect_left(self._lengths, size) if i == len(self._lengths): length = self._roundup(max(self._size, size), mmap.PAGESIZE) self._size *= 2 util.info('allocating a new mmap of length %d', length) arena = Arena(length) self._arenas.append(arena) return (arena, 0, length) else: length = self._lengths[i] seq = self._len_to_seq[length] block = seq.pop() if not seq: del self._len_to_seq[length], self._lengths[i] (arena, start, stop) = block del self._start_to_block[(arena, start)] del self._stop_to_block[(arena, stop)] return block def _free(self, block): # free location and try to merge with neighbours (arena, start, stop) = block try: prev_block = self._stop_to_block[(arena, start)] except KeyError: pass else: start, _ = self._absorb(prev_block) try: next_block = self._start_to_block[(arena, stop)] except KeyError: pass else: _, stop = self._absorb(next_block) block = (arena, start, stop) length = stop - start try: self._len_to_seq[length].append(block) except KeyError: self._len_to_seq[length] = [block] bisect.insort(self._lengths, length) self._start_to_block[(arena, start)] = block self._stop_to_block[(arena, stop)] = block def _absorb(self, block): # deregister this block so it can be merged with a neighbour (arena, start, stop) = block del self._start_to_block[(arena, start)] del self._stop_to_block[(arena, stop)] length = stop - start seq = self._len_to_seq[length] seq.remove(block) if not seq: del self._len_to_seq[length] self._lengths.remove(length) return start, stop def _free_pending_blocks(self): # Free all the blocks in the pending list - called with the lock held. while True: try: block = self._pending_free_blocks.pop() except IndexError: break self._allocated_blocks.remove(block) self._free(block) def free(self, block): # free a block returned by malloc() # Since free() can be called asynchronously by the GC, it could happen # that it's called while self._lock is held: in that case, # self._lock.acquire() would deadlock (issue #12352). To avoid that, a # trylock is used instead, and if the lock can't be acquired # immediately, the block is added to a list of blocks to be freed # synchronously sometimes later from malloc() or free(), by calling # _free_pending_blocks() (appending and retrieving from a list is not # strictly thread-safe but under cPython it's atomic thanks to the GIL). assert os.getpid() == self._lastpid if not self._lock.acquire(False): # can't acquire the lock right now, add the block to the list of # pending blocks to free self._pending_free_blocks.append(block) else: # we hold the lock try: self._free_pending_blocks() self._allocated_blocks.remove(block) self._free(block) finally: self._lock.release() def malloc(self, size): # return a block of right size (possibly rounded up) assert 0 <= size < sys.maxsize if os.getpid() != self._lastpid: self.__init__() # reinitialize after fork with self._lock: self._free_pending_blocks() size = self._roundup(max(size,1), self._alignment) (arena, start, stop) = self._malloc(size) new_stop = start + size if new_stop < stop: self._free((arena, new_stop, stop)) block = (arena, start, new_stop) self._allocated_blocks.add(block) return block # # Class representing a chunk of an mmap -- can be inherited by child process # class BufferWrapper(object): _heap = Heap() def __init__(self, size): assert 0 <= size < sys.maxsize block = BufferWrapper._heap.malloc(size) self._state = (block, size) util.Finalize(self, BufferWrapper._heap.free, args=(block,)) def create_memoryview(self): (arena, start, stop), size = self._state return memoryview(arena.buffer)[start:start+size] __pycache__/pool.cpython-36.opt-1.pyc 0000644 00000050126 15204243002 0013306 0 ustar 00 3 \�e � @ s8 d dgZ ddlZddlZddlZddlZddlZddlZddlZddlm Z ddlm Z m Z dZ dZ dZej� Zdd � Zd d � ZG d d � d e�ZG dd� d�Zdd� ZG dd� de�Zdf ddfdd�Zdd� ZG dd � d e�ZG dd� de�Z e Z G d d � d e �Z G d d � d e�Z G d d!� d!e �Z G d"d� de�Z!dS )#�Pool� ThreadPool� N� )�util)� get_context� TimeoutError� c C s t t| � �S )N)�list�map)�args� r �,/usr/lib64/python3.6/multiprocessing/pool.py�mapstar+ s r c C s t tj| d | d ��S )Nr r )r � itertools�starmap)r r r r � starmapstar. s r c @ s e Zd Zdd� Zdd� ZdS )�RemoteTracebackc C s || _ d S )N)�tb)�selfr r r r �__init__6 s zRemoteTraceback.__init__c C s | j S )N)r )r r r r �__str__8 s zRemoteTraceback.__str__N)�__name__� __module__� __qualname__r r r r r r r 5 s r c @ s e Zd Zdd� Zdd� ZdS )�ExceptionWithTracebackc C s0 t jt|�||�}dj|�}|| _d| | _d S )N� z """ %s""")� traceback�format_exception�type�join�excr )r r r r r r r < s z ExceptionWithTraceback.__init__c C s t | j| jffS )N)� rebuild_excr r )r r r r � __reduce__A s z!ExceptionWithTraceback.__reduce__N)r r r r r" r r r r r ; s r c C s t |�| _| S )N)r � __cause__)r r r r r r! D s r! c s0 e Zd ZdZ� fdd�Zdd� Zdd� Z� ZS )�MaybeEncodingErrorzVWraps possible unpickleable errors, so they can be safely sent through the socket.c s. t |�| _t |�| _tt| �j| j| j� d S )N)�reprr �value�superr$ r )r r r& )� __class__r r r P s zMaybeEncodingError.__init__c C s d| j | jf S )Nz(Error sending result: '%s'. Reason: '%s')r& r )r r r r r U s zMaybeEncodingError.__str__c C s d| j j| f S )Nz<%s: %s>)r( r )r r r r �__repr__Y s zMaybeEncodingError.__repr__)r r r �__doc__r r r) � __classcell__r r )r( r r$ L s r$ Fc ' C s� |j }| j}t| d�r*| jj� |jj� |d k r:||� d}�xH|d ksX|oT||k �r�y |� } W n$ ttfk r� tj d� P Y nX | d kr�tj d� P | \} } } } }yd| | |�f}W nB t k r� } z&|r�| t k r�t ||j �}d|f}W Y d d }~X nX y|| | |f� W nR t k �rd } z4t||d �}tj d| � || | d|ff� W Y d d }~X nX d } } } } } }|d7 }qBW tj d | � d S ) N�_writerr z)worker got EOFError or OSError -- exitingz worker got sentinel -- exitingTFr z0Possible encoding error while sending result: %sz worker exiting after %d tasks)�put�get�hasattrr, �close�_reader�EOFError�OSErrorr �debug� Exception�_helper_reraises_exceptionr � __traceback__r$ )�inqueue�outqueue� initializer�initargsZmaxtasksZwrap_exceptionr- r. Z completed�task�job�i�funcr �kwds�result�e�wrappedr r r �worker] sD $ rD c C s | �dS )z@Pickle-able helper function for use by _guarded_task_generation.Nr )Zexr r r r6 � s r6 c @ s6 e Zd ZdZdZdd� Zddf ddfdd�Zdd � Zd d � Zd d � Z dd� Z f i fdd�Z d=dd�Z d>dd�Z d?dd�Zdd� Zd@dd �ZdAd d �Zf i ddfd d �ZdBd!d"�ZdCd#d$�Zed%d&� �Zed'd(� �Zed)d*� �Zed+d,� �Zd-d.� Zd/d0� Zd1d2� Z d3d4� Z ed5d6� �Z e d7d8� �Z d9d:� Z!d;d<� Z"dS )Dr zS Class which supports an async version of applying functions to arguments. Tc O s | j j||�S )N)�_ctx�Process)r r r@ r r r rF � s z Pool.ProcessNc C sn |pt � | _| j� tj� | _i | _t| _|| _ || _ || _ |d krPt j � pNd}|dk r`td��|d k rzt|� rztd��|| _g | _| j� tjtj| fd�| _d| j_t| j_| jj� tjtj| j| j | j | j| jfd�| _ d| j _t| j _| j j� tjtj | j | j | jfd�| _!d| j!_t| j!_| j!j� t"j#| | j$| j| j%| j | j| j| j | j!| jfdd�| _&d S )Nr z&Number of processes must be at least 1z initializer must be a callable)�targetr T� )r Z exitpriority)'r rE � _setup_queues�queue�Queue� _taskqueue�_cache�RUN�_state�_maxtasksperchild� _initializer� _initargs�os� cpu_count� ValueError�callable� TypeError� _processes�_pool�_repopulate_pool� threadingZThreadr �_handle_workers�_worker_handler�daemon�start� _handle_tasks� _quick_put� _outqueue� _task_handler�_handle_results� _quick_get�_result_handlerr ZFinalize�_terminate_pool�_inqueue� _terminate)r � processesr: r; Zmaxtasksperchild�contextr r r r � sT z Pool.__init__c C sZ d}xPt tt| j���D ]:}| j| }|jdk rtjd| � |j� d}| j|= qW |S )z�Cleanup after any worker processes which have exited due to reaching their specified lifetime. Returns True if any workers were cleaned up. FNzcleaning up worker %dT)�reversed�range�lenrY �exitcoder r4 r )r Zcleanedr> rD r r r �_join_exited_workers� s zPool._join_exited_workersc C s| xvt | jt| j� �D ]^}| jt| j| j| j| j | j | j fd�}| jj |� |j jdd�|_ d|_|j� tjd� qW dS )z�Bring the number of pool processes up to the specified number, for use after reaping workers which have exited. )rG r rF Z PoolWorkerTz added workerN)rm rX rn rY rF rD rh rb rQ rR rP �_wrap_exception�append�name�replacer^ r_ r r4 )r r> �wr r r rZ � s zPool._repopulate_poolc C s | j � r| j� dS )zEClean up any exited workers and start replacements for them. N)rp rZ )r r r r �_maintain_pool� s zPool._maintain_poolc C s4 | j j� | _| j j� | _| jjj| _| jjj| _ d S )N) rE Z SimpleQueuerh rb r, �sendra r1 �recvre )r r r r rI � s zPool._setup_queuesc C s | j |||�j� S )z6 Equivalent of `func(*args, **kwds)`. )� apply_asyncr. )r r? r r@ r r r �apply� s z Pool.applyc C s | j ||t|�j� S )zx Apply `func` to each element in `iterable`, collecting the results in a list that is returned. )� _map_asyncr r. )r r? �iterable� chunksizer r r r s zPool.mapc C s | j ||t|�j� S )z� Like `map()` method but the elements of the `iterable` are expected to be iterables as well and will be unpacked as arguments. Hence `func` and (a, b) becomes func(a, b). )r{ r r. )r r? r| r} r r r r s z Pool.starmapc C s | j ||t|||�S )z= Asynchronous version of `starmap()` method. )r{ r )r r? r| r} �callback�error_callbackr r r � starmap_async s zPool.starmap_asyncc c sn y0d}x&t |�D ]\}}||||fi fV qW W n8 tk rh } z ||d t|fi fV W Y dd}~X nX dS )z�Provides a generator of tasks for imap and imap_unordered with appropriate handling for iterables which throw exceptions during iteration.r N���)� enumerater5 r6 )r Z result_jobr? r| r> �xrB r r r �_guarded_task_generation s z Pool._guarded_task_generationr c C s� | j tkrtd��|dkrFt| j�}| jj| j|j||�|j f� |S t j |||�}t| j�}| jj| j|jt |�|j f� dd� |D �S dS )zP Equivalent of `map()` -- can be MUCH slower than `Pool.map()`. zPool not runningr c s s | ]}|D ] }|V q qd S )Nr )�.0�chunk�itemr r r � <genexpr>@ s z Pool.imap.<locals>.<genexpr>N) rO rN rU � IMapIteratorrM rL r- r� �_job� _set_lengthr � _get_tasksr )r r? r| r} rA � task_batchesr r r �imap' s z Pool.imapc C s� | j tkrtd��|dkrFt| j�}| jj| j|j||�|j f� |S t j |||�}t| j�}| jj| j|jt |�|j f� dd� |D �S dS )zL Like `imap()` method but ordering of results is arbitrary. zPool not runningr c s s | ]}|D ] }|V q qd S )Nr )r� r� r� r r r r� [ s z&Pool.imap_unordered.<locals>.<genexpr>N) rO rN rU �IMapUnorderedIteratorrM rL r- r� r� r� r r� r )r r? r| r} rA r� r r r �imap_unorderedB s zPool.imap_unorderedc C sB | j tkrtd��t| j||�}| jj|jd|||fgdf� |S )z; Asynchronous version of `apply()` method. zPool not runningr N)rO rN rU � ApplyResultrM rL r- r� )r r? r r@ r~ r rA r r r ry ] s zPool.apply_asyncc C s | j ||t|||�S )z9 Asynchronous version of `map()` method. )r{ r )r r? r| r} r~ r r r r � map_asynch s zPool.map_asyncc C s� | j tkrtd��t|d�s$t|�}|dkrTtt|�t| j�d �\}}|rT|d7 }t|�dkrdd}tj |||�}t | j |t|�||d�} | j j | j| j||�df� | S )zY Helper function to implement map, starmap and their async counterparts. zPool not running�__len__N� r r )r )rO rN rU r/ r �divmodrn rY r r� � MapResultrM rL r- r� r� ) r r? r| Zmapperr} r~ r Zextrar� rA r r r r{ p s& zPool._map_asyncc C sT t j� }x0|jtks$| jr8|jtkr8| j� tjd� q W | j j d � t j d� d S )Ng�������?zworker handler exiting) r[ �current_threadrO rN rM � TERMINATErv �time�sleeprL r- r r4 )�pool�threadr r r r\ � s zPool._handle_workersc C sj t j� }x�t| jd �D ]�\}}d }z�x�|D ]�}|jr@tjd� P y ||� W q* tk r� } zD|d d� \} } y|| j| d| f� W n t k r� Y nX W Y d d } ~ X q*X q*W |r�tjd� |r�|d nd } || d � wP W d d } }} X qW tjd� y:tjd� |j d � tjd� x|D ]} |d � �q&W W n t k �rZ tjd � Y nX tjd � d S ) Nz'task handler found thread._state != RUNr Fzdoing set_length()r ztask handler got sentinelz/task handler sending sentinel to result handlerz(task handler sending sentinel to workersz/task handler got OSError when sending sentinelsztask handler exitingr� ) r[ r� �iterr. rO r r4 r5 �_set�KeyErrorr- r3 ) � taskqueuer- r9 r� �cacher� ZtaskseqZ set_lengthr< rB r= �idx�pr r r r` � sB zPool._handle_tasksc C s� t j� }x�y |� }W n" ttfk r6 tjd� d S X |jrJtjd� P |d kr^tjd� P |\}}}y|| j||� W n tk r� Y nX d } }}q W x�|o�|jt k�r:y |� }W n" ttfk r� tjd� d S X |d kr�tjd� q�|\}}}y|| j||� W n tk �r* Y nX d } }}q�W t | d��r�tjd� y,x&t d�D ]}| j j � �spP |� �q^W W n ttfk �r� Y nX tjdt|�|j� d S ) Nz.result handler got EOFError/OSError -- exitingz,result handler found thread._state=TERMINATEzresult handler got sentinelz&result handler ignoring extra sentinelr1 z"ensuring that outqueue is not full� z7result handler exiting: len(cache)=%s, thread._state=%s)r[ r� r3 r2 r r4 rO r� r� r� r/ rm r1 �pollrn )r9 r. r� r� r< r= r> �objr r r rd � sZ zPool._handle_resultsc c s4 t |�}x&ttj||��}|s"d S | |fV q W d S )N)r� �tupler �islice)r? �it�sizer� r r r r� s zPool._get_tasksc C s t d��d S )Nz:pool objects cannot be passed between processes or pickled)�NotImplementedError)r r r r r" s zPool.__reduce__c C s& t jd� | jtkr"t| _t| j_d S )Nz closing pool)r r4 rO rN �CLOSEr] )r r r r r0 s z Pool.closec C s$ t jd� t| _t| j_| j� d S )Nzterminating pool)r r4 r� rO r] ri )r r r r � terminate s zPool.terminatec C sD t jd� | jj� | jj� | jj� x| jD ] }|j� q0W d S )Nz joining pool)r r4 r] r rc rf rY )r r� r r r r s z Pool.joinc C sD t jd� | jj� x*|j� r>| jj� r>| jj� tj d� qW d S )Nz7removing tasks from inqueue until task handler finishedr ) r r4 Z_rlock�acquire�is_aliver1 r� rx r� r� )r8 � task_handlerr� r r r �_help_stuff_finish( s zPool._help_stuff_finishc C s( t jd� t|_t|_t jd� | j||t|�� t|_|jd � t jd� tj� |k r`|j � |r�t |d d�r�t jd� x |D ]} | j d kr�| j � q�W t jd� tj� |k r�|j � t jd� tj� |k r�|j � |o�t |d d��r$t jd � x,|D ]$} | j � r�t jd | j � | j � q�W d S ) Nzfinalizing poolz&helping task handler/workers to finishzjoining worker handlerr r� zterminating workerszjoining task handlerzjoining result handlerzjoining pool workerszcleaning up worker %d)r r4 r� rO r� rn r- r[ r� r r/ ro r� r� �pid) �clsr� r8 r9 r� Zworker_handlerr� Zresult_handlerr� r� r r r rg 1 s6 zPool._terminate_poolc C s | S )Nr )r r r r � __enter___ s zPool.__enter__c C s | j � d S )N)r� )r �exc_typeZexc_valZexc_tbr r r �__exit__b s z Pool.__exit__)N)N)NNN)r )r )NNN)NNN)#r r r r* rq rF r rp rZ rv rI rz r r r� r� r� r� ry r� r{ � staticmethodr\ r` rd r� r" r0 r� r r� � classmethodrg r� r� r r r r r � sF 8 . < .c @ s@ e Zd Zdd� Zdd� Zdd� Zddd �Zdd d �Zd d � ZdS )r� c C s4 t j� | _tt�| _|| _|| _|| _| || j< d S )N) r[ ZEvent�_event�next� job_counterr� rM � _callback�_error_callback)r r� r~ r r r r r k s zApplyResult.__init__c C s | j j� S )N)r� Zis_set)r r r r �readys s zApplyResult.readyc C s | j S )N)�_success)r r r r � successfulv s zApplyResult.successfulNc C s | j j|� d S )N)r� �wait)r �timeoutr r r r� z s zApplyResult.waitc C s, | j |� | j� st�| jr"| jS | j�d S )N)r� r� r r� �_value)r r� r r r r. } s zApplyResult.getc C sV |\| _ | _| jr$| j r$| j| j� | jr>| j r>| j| j� | jj� | j| j= d S )N)r� r� r� r� r� �setrM r� )r r> r� r r r r� � s zApplyResult._set)N)N) r r r r r� r� r� r. r� r r r r r� i s r� c @ s e Zd Zdd� Zdd� ZdS )r� c C sf t j| |||d� d| _d g| | _|| _|dkrLd| _| jj� || j= n|| t || � | _d S )N)r Tr ) r� r r� r� � _chunksize� _number_leftr� r� r� �bool)r r� r} �lengthr~ r r r r r � s zMapResult.__init__c C s� | j d8 _ |\}}|rp| jrp|| j|| j |d | j �< | j dkr�| jrZ| j| j� | j| j= | jj� nH| r�| jr�d| _|| _| j dkr�| j r�| j | j� | j| j= | jj� d S )Nr r F) r� r� r� r� r� rM r� r� r� r� )r r> Zsuccess_result�successrA r r r r� � s" zMapResult._setN)r r r r r� r r r r r� � s r� c @ s: e Zd Zdd� Zdd� Zd dd�ZeZdd � Zd d � ZdS ) r� c C sJ t jt j� �| _tt�| _|| _tj � | _ d| _ d | _ i | _ | || j< d S )Nr )r[ Z ConditionZLock�_condr� r� r� rM � collections�deque�_items�_index�_length� _unsorted)r r� r r r r � s zIMapIterator.__init__c C s | S )Nr )r r r r �__iter__� s zIMapIterator.__iter__Nc C s� | j �~ y| jj� }W nh tk r~ | j| jkr6t�| j j|� y| jj� }W n( tk rx | j| jkrpt�t�Y nX Y nX W d Q R X |\}}|r�|S |�d S )N) r� r� �popleft� IndexErrorr� r� � StopIterationr� r )r r� r� r� r& r r r r� � s" zIMapIterator.nextc C s� | j �� | j|krr| jj|� | jd7 _x8| j| jkrd| jj| j�}| jj|� | jd7 _q.W | j j� n || j|< | j| jkr�| j| j = W d Q R X d S )Nr ) r� r� r� rr r� �pop�notifyr� rM r� )r r> r� r r r r� � s zIMapIterator._setc C s< | j �, || _| j| jkr.| j j� | j| j= W d Q R X d S )N)r� r� r� r� rM r� )r r� r r r r� � s zIMapIterator._set_length)N) r r r r r� r� �__next__r� r� r r r r r� � s r� c @ s e Zd Zdd� ZdS )r� c C sP | j �@ | jj|� | jd7 _| j j� | j| jkrB| j| j= W d Q R X d S )Nr )r� r� rr r� r� r� rM r� )r r> r� r r r r� s zIMapUnorderedIterator._setN)r r r r� r r r r r� � s r� c @ s@ e Zd ZdZedd� �Zddf fdd�Zdd� Zed d � �ZdS ) r Fc O s ddl m} || |�S )Nr )rF )ZdummyrF )r r@ rF r r r rF s zThreadPool.ProcessNc C s t j| |||� d S )N)r r )r rj r: r; r r r r s zThreadPool.__init__c C s, t j� | _t j� | _| jj| _| jj| _d S )N)rJ rK rh rb r- ra r. re )r r r r rI s zThreadPool._setup_queuesc C s< | j �, | jj� | jjd g| � | j j� W d Q R X d S )N)Z not_emptyrJ �clear�extendZ notify_all)r8 r� r� r r r r� s z ThreadPool._help_stuff_finish) r r r rq r� rF r rI r� r r r r r s )"�__all__r[ rJ r r� rS r� r r r r r rN r� r� �countr� r r r5 r r r! r$ rD r6 �objectr r� Z AsyncResultr� r� r� r r r r r �<module> s@ * \&)@ __pycache__/__init__.cpython-36.opt-2.pyc 0000644 00000000773 15204243002 0014100 0 ustar 00 3 \� � @ s\ d dl Z ddlmZ e� jdd� ejjD �� ejjZdZdZde j krXe j d e j d < dS ) � N� )�contextc c s | ]}|t tj|�fV qd S )N)�getattrr �_default_context)�.0�name� r �0/usr/lib64/python3.6/multiprocessing/__init__.py� <genexpr> s r � � �__main__Z __mp_main__) �sys� r �globals�updater �__all__ZSUBDEBUGZ SUBWARNING�modulesr r r r �<module> s __pycache__/semaphore_tracker.cpython-36.pyc 0000644 00000007125 15204243002 0015075 0 ustar 00 3 \ � @ s� d dl Z d dlZd dlZd dlZd dlZd dlZddlmZ ddlmZ dddgZ G dd � d e �Z e � Z e j Z e jZe jZe jZd d � ZdS ) � N� )�spawn)�util�ensure_running�register� unregisterc @ s< e Zd Zdd� Zdd� Zdd� Zdd� Zd d � Zd d � Zd S )�SemaphoreTrackerc C s t j� | _d | _d | _d S )N)� threadingZLock�_lock�_fd�_pid)�self� r �9/usr/lib64/python3.6/multiprocessing/semaphore_tracker.py�__init__ s zSemaphoreTracker.__init__c C s | j � | jS )N)r r )r r r r �getfd" s zSemaphoreTracker.getfdc C s | j �� | jdk rRtj| jtj�\}}|s0dS tj| j� d| _d| _tjd� g }y|j t j j � � W n t k r~ Y nX d}tj� \}}zlyB|j |� tj� }|gtj� }|d|| g7 }tj|||�}W n tj|� � Y nX || _|| _W dtj|� X W dQ R X dS )z�Make sure that semaphore tracker process is running. This can be run from any process. Usually a child process will use the semaphore created by its parent.NzWsemaphore_tracker: process died unexpectedly, relaunching. Some semaphores might leak.z;from multiprocessing.semaphore_tracker import main;main(%d)z-c)r r �os�waitpid�WNOHANG�closer �warnings�warn�append�sys�stderr�fileno� Exception�piper Zget_executabler Z _args_from_interpreter_flagsZspawnv_passfds) r �pidZstatusZ fds_to_pass�cmd�r�wZexe�argsr r r r & s8 z SemaphoreTracker.ensure_runningc C s | j d|� dS )z2Register name of semaphore with semaphore tracker.ZREGISTERN)�_send)r �namer r r r Q s zSemaphoreTracker.registerc C s | j d|� dS )z4Unregister name of semaphore with semaphore tracker.Z UNREGISTERN)r# )r r$ r r r r U s zSemaphoreTracker.unregisterc C sP | j � dj||�jd�}t|�dkr.td��tj| j|�}|t|�ksLt�d S )Nz{0}:{1} �asciii z name too long) r �format�encode�len� ValueErrorr �writer �AssertionError)r r r$ �msg�nbytesr r r r# Y s zSemaphoreTracker._sendN) �__name__� __module__� __qualname__r r r r r r# r r r r r s +r c - C s� t j t jt j� t j t jt j� x6tjtjfD ]&}y |j� W q. tk rR Y q.X q.W t � }z�t | d���}x�|D ]�}yJ|j � j d�\}}|dkr�|j |� n |dkr�|j|� n td| ��W qr tk r� ytjtj� � W n Y nX Y qrX qrW W dQ R X W d|�r<ytjdt|� � W n tk �r: Y nX xf|D ]^}zT|jd�}ytj|� W n6 tk �r� } ztjd ||f � W Y dd}~X nX W dX �qBW X dS ) zRun semaphore tracker.�rb� :s REGISTERs UNREGISTERzunrecognized command %rNzRsemaphore_tracker: There appear to be %d leaked semaphores to clean up at shutdownr% zsemaphore_tracker: %r: %s)�signal�SIGINT�SIG_IGN�SIGTERMr �stdin�stdoutr r �set�open�strip�split�add�remove� RuntimeError� excepthook�exc_infor r r( �decode�_multiprocessingZ sem_unlink)�fd�f�cache�liner r$ �er r r �maink sJ (rI )r r3 r r r rC � r r �__all__�objectr Z_semaphore_trackerr r r r rI r r r r �<module> s I __pycache__/sharedctypes.cpython-36.opt-2.pyc 0000644 00000014650 15204243002 0015036 0 ustar 00 3 \e � @ s: d dl Z d dlZddlmZ ddlmZ ddlmZmZ ejZ dddd d d gZ e j e j e j e je je je je je je je je jd � Zd d� Zdd� Zdd� Zddd�dd�Zddd�dd �Z dd � Z d&dd �Z dd� Z dd� Z dd � Z!d Z"i Z#ej$� Z%G d d � d e&�Z'G d d!� d!e'�Z(G d"d#� d#e'�Z)G d$d%� d%e)�Z*dS )'� N� )�heap)� get_context)� reduction�assert_spawning�RawValue�RawArray�Value�Array�copy� synchronized) �c�u�b�B�h�H�i�I�l�L�f�dc C s t j| �}tj|�}t| |d �S )N)�ctypes�sizeofr Z BufferWrapper� rebuild_ctype)�type_�size�wrapper� r �4/usr/lib64/python3.6/multiprocessing/sharedctypes.py� _new_value& s r! c G s< t j| | �}t|�}tjtj|�dtj|�� |j|� |S )Nr )�typecode_to_type�getr! r �memset� addressofr �__init__)�typecode_or_type�argsr �objr r r r + s c C sj t j| | �}t|t�rD|| }t|�}tjtj|�dtj|�� |S |t |� }t|�}|j |� |S d S )Nr ) r" r# � isinstance�intr! r r$ r% r �lenr& )r' �size_or_initializerr r) �resultr r r r 5 s T)�lock�ctxc G sX t | f|�� }|dkr|S |dkr4|p*t� }|j� }t|d�sJtd| ��t|||d�S )NFT�acquirez '%r' has no method 'acquire')r0 )TN)r r �RLock�hasattr�AttributeErrorr )r' r/ r0 r( r) r r r r E s c C sT t | |�}|dkr|S |dkr0|p&t� }|j� }t|d�sFtd| ��t|||d�S )NFTr1 z '%r' has no method 'acquire')r0 )TN)r r r2 r3 r4 r )r' r- r/ r0 r) r r r r S s c C s t t| ��}| tj|�d< |S )Nr )r! �typer Zpointer)r) Znew_objr r r r a s c C s� |pt � }t| tj�r"t| ||�S t| tj�rR| jtjkrFt| ||�S t | ||�S t | �}y t | }W nV t k r� dd� |j D �}tdd� |D ��}d|j }t |tf|� }t |< Y nX || ||�S d S )Nc S s g | ] }|d �qS )r r )�.0Zfieldr r r � <listcomp>u s z synchronized.<locals>.<listcomp>c s s | ]}|t |�fV qd S )N)� make_property)r6 �namer r r � <genexpr>v s z synchronized.<locals>.<genexpr>� Synchronized)r r* r Z _SimpleCDatar; r �_type_�c_char�SynchronizedString�SynchronizedArrayr5 � class_cache�KeyErrorZ_fields_�dict�__name__�SynchronizedBase)r) r/ r0 �cls�scls�namesr Z classnamer r r r f s c C s@ t | � t| tj�r(t| j| j| jffS tt| �| jd ffS d S )N) r r* r r r r< �_wrapperZ_length_r5 )r) r r r � reduce_ctype s rI c C s8 |d k r| | } t j| t� |j� }| j|�}||_|S )N)�_ForkingPickler�registerrI Zcreate_memoryviewZ from_bufferrH )r r ZlengthZbufr) r r r r � s r c C sJ yt | S tk rD i }tt| fd |� || t | < || S X d S )N� )� prop_cacherA �exec�template)r9 r r r r r8 � s r8 z� def get%s(self): self.acquire() try: return self._obj.%s finally: self.release() def set%s(self, value): self.acquire() try: self._obj.%s = value finally: self.release() %s = property(get%s, set%s) c @ sF e Zd Zddd�Zdd� Zdd� Zdd � Zd d � Zd d � Zdd� Z dS )rD Nc C sB || _ |r|| _n|p tdd�}|j� | _| jj| _| jj| _d S )NT)Zforce)�_obj�_lockr r2 r1 �release)�selfr) r/ r0 r r r r& � s zSynchronizedBase.__init__c C s | j j� S )N)rQ � __enter__)rS r r r rT � s zSynchronizedBase.__enter__c G s | j j|� S )N)rQ �__exit__)rS r( r r r rU � s zSynchronizedBase.__exit__c C s t | � t| j| jffS )N)r r rP rQ )rS r r r � __reduce__� s zSynchronizedBase.__reduce__c C s | j S )N)rP )rS r r r �get_obj� s zSynchronizedBase.get_objc C s | j S )N)rQ )rS r r r �get_lock� s zSynchronizedBase.get_lockc C s dt | �j| jf S )Nz<%s wrapper for %s>)r5 rC rP )rS r r r �__repr__� s zSynchronizedBase.__repr__)NN) rC � __module__� __qualname__r& rT rU rV rW rX rY r r r r rD � s rD c @ s e Zd Zed�ZdS )r; �valueN)rC rZ r[ r8 r\ r r r r r; � s r; c @ s4 e Zd Zdd� Zdd� Zdd� Zdd� Zd d � Zd S ) r? c C s t | j�S )N)r, rP )rS r r r �__len__� s zSynchronizedArray.__len__c C s | � | j | S Q R X d S )N)rP )rS r r r r � __getitem__� s z SynchronizedArray.__getitem__c C s | � || j |< W d Q R X d S )N)rP )rS r r\ r r r � __setitem__� s z SynchronizedArray.__setitem__c C s | � | j ||� S Q R X d S )N)rP )rS �start�stopr r r � __getslice__� s z SynchronizedArray.__getslice__c C s" | � || j ||�< W d Q R X d S )N)rP )rS r` ra �valuesr r r � __setslice__� s z SynchronizedArray.__setslice__N)rC rZ r[ r] r^ r_ rb rd r r r r r? � s r? c @ s e Zd Zed�Zed�ZdS )r> r\ �rawN)rC rZ r[ r8 r\ re r r r r r> � s r> )NN)+r �weakref� r r �contextr r ZForkingPicklerrJ �__all__r= Zc_wcharZc_byteZc_ubyteZc_shortZc_ushortZc_intZc_uintZc_longZc_ulongZc_floatZc_doubler" r! r r r r r r rI r r8 rO rM �WeakKeyDictionaryr@ �objectrD r; r? r> r r r r �<module> s: __pycache__/reduction.cpython-36.opt-2.pyc 0000644 00000015662 15204243002 0014340 0 ustar 00 3 \ $ � @ s d dl mZmZ d dlZd dlZd dlZd dlZd dlZd dlZd dl Z ddl m Z ddddd gZ e j d kp�eed �o�eed �o�eejd �ZG dd� dej�ZejZd5dd �Ze j d k�re dddg7 Z d dlZd6dd�Zdd� Zdd� Zdd� ZG dd� de�ZnHe dddg7 Z d dlZe j d kZ d d� Z d d� Z d d� Zd d� Zd!d� Z d"d#� Z G d$d%� d%�Z!ee"e!� j#�e � d&d'� Z$ee"e%j&�e$� ee"e'j(�e$� d(d)� Z)d*d+� Z*eej+e)� e j d k�r�d,d-� Z,d.d/� Z-eeje,� n d0d-� Z,d1d/� Z-eeje,� G d2d3� d3ed4�Z.dS )7� )�ABCMeta�abstractmethodN� )�context� send_handle� recv_handle�ForkingPickler�register�dump�win32ZCMSG_LEN� SCM_RIGHTS�sendmsgc sF e Zd Zi ZejZ� fdd�Zedd� �Z eddd��Z e j Z � Z S ) r c s* t � j|� | jj� | _| jj| j� d S )N)�super�__init__�_copyreg_dispatch_table�copy�dispatch_table�update�_extra_reducers)�self�args)� __class__� �1/usr/lib64/python3.6/multiprocessing/reduction.pyr & s zForkingPickler.__init__c C s || j |< d S )N)r )�cls�type�reducer r r r + s zForkingPickler.registerNc C s t j� }| ||�j|� |j� S )N)�io�BytesIOr � getbuffer)r �obj�protocolZbufr r r �dumps0 s zForkingPickler.dumps)N)�__name__� __module__� __qualname__r �copyregr r r � classmethodr r" �pickle�loads� __classcell__r r )r r r ! s c C s t ||�j| � d S )N)r r )r �filer! r r r r : s � DupHandle� duplicate� steal_handleFc C s* |d krt j� }t jt j� | |d|t j�S )Nr )�_winapi�GetCurrentProcess�DuplicateHandle�DUPLICATE_SAME_ACCESS)�handleZtarget_processZ inheritabler r r r- G s c C sB t jt jd| �}z t j||t j� ddt jt jB �S t j|� X d S )NFr )r/ � OpenProcess�PROCESS_DUP_HANDLEr1 r0 r2 �DUPLICATE_CLOSE_SOURCE� CloseHandle)Z source_pidr3 Zsource_process_handler r r r. O s c C s t |tj|�}| j|� d S )N)r, r/ r2 �send)�connr3 �destination_pidZdhr r r r [ s c C s | j � j� S )N)�recv�detach)r9 r r r r ` s c @ s e Zd Zddd�Zdd� ZdS )r, Nc C s\ |d krt j� }tjtjd|�}z tjtj� |||dd�| _W d tj|� X || _ || _ d S )NFr ) �os�getpidr/ r4 r5 r1 r0 �_handler7 �_access�_pid)r r3 �access�pid�procr r r r f s zDupHandle.__init__c C sV | j tj� kr| jS tjtjd| j �}z tj|| jtj� | j dtj �S tj |� X d S )NF) rA r= r> r? r/ r4 r5 r1 r0 r@ r6 r7 )r rD r r r r< u s zDupHandle.detach)N)r# r$ r% r r< r r r r r, d s �DupFd�sendfds�recvfds�darwinc C sV t j d|�}tt|�d g�}| j|gtjtj|fg� trR| jd�dkrRt d��d S )N�i� r � Az%did not receive acknowledgement of fd) �array�bytes�lenr �socket� SOL_SOCKETr � ACKNOWLEDGEr; � RuntimeError)�sockZfds�msgr r r rF � s c C s� t j d�}|j| }| jdtj|��\}}}}| r>| r>t�yxtrN| jd� t|�dkrjt dt|� ��|d \}} } |tj kr�| tj kr�t| �|j dkr�t �|j | � t|�S W n t tfk r� Y nX t d��d S )NrI r rK z received %d items of ancdatar zInvalid data received)rL �itemsizeZrecvmsgrO Z CMSG_SPACE�EOFErrorrQ r8 rN rR rP r � ValueErrorZ frombytes�list� IndexError) rS �size�aZ bytes_sizerT Zancdata�flagsZaddrZ cmsg_levelZ cmsg_typeZ cmsg_datar r r rG � s* c C s2 t j| j� t jt j��}t||g� W d Q R X d S )N)rO �fromfd�fileno�AF_UNIX� SOCK_STREAMrF )r9 r3 r: �sr r r r � s c C s0 t j| j� t jt j��}t|d�d S Q R X d S )Nr r )rO r] r^ r_ r` rG )r9 ra r r r r � s c C sF t j� }|d k r |j|j| ��S tr:ddlm} |j| �S td��d S )Nr )�resource_sharerz&SCM_RIGHTS appears not to be available)r Zget_spawning_popenrE Zduplicate_for_child�HAVE_SEND_HANDLE� rb rW )�fdZ popen_objrb r r r rE � s c C s2 | j d kr t| j| jjffS t| j | jjffS d S )N)�__self__�getattrr �__func__r# )�mr r r �_reduce_method� s rj c @ s e Zd Zdd� ZdS )�_Cc C s d S )Nr )r r r r �f� s z_C.fN)r# r$ r% rl r r r r rk � s rk c C s t | j| jffS )N)rg � __objclass__r# )ri r r r �_reduce_method_descriptor� s rn c C s t | j| j| jpi ffS )N)�_rebuild_partial�funcr �keywords)�pr r r �_reduce_partial� s rs c C s t j| f|�|�S )N)� functools�partial)rp r rq r r r ro � s ro c C s ddl m} t|| �ffS )Nr )� DupSocket)rb rv �_rebuild_socket)ra rv r r r �_reduce_socket� s rx c C s | j � S )N)r< )Zdsr r r rw � s rw c C s" t | j� �}t|| j| j| jffS )N)rE r^ rw �familyr �proto)ra �dfr r r rx � s c C s | j � }tj||||d�S )N)r^ )r< rO )r{ ry r rz re r r r rw � s c @ s` e Zd ZeZeZeZeZeZej dkr4e Z e Z e Z n e Z eZeZeZeZeZeZeZdd� ZdS )�AbstractReducerr c G sN t tt� j�t� t ttj�t� t ttj �t� t t j t � t t j t� d S )N)r r rk rl rj rX �appendrn �int�__add__rt ru rs rO rx )r r r r r r s zAbstractReducer.__init__N)r# r$ r% r r r r r �sys�platformr. r- r, rF rG rE rj rn ro rx rw r r r r r r| � s$ r| )� metaclass)N)NF)/�abcr r r& rt r r= r( rO r� rd r �__all__r� �hasattrrc ZPicklerr r r r/ r- r. r r �objectr, rL rQ rF rG rE rj rk r rl rn rX r} r~ r rs ro ru rx rw r| r r r r �<module> sb # __pycache__/resource_sharer.cpython-36.opt-1.pyc 0000644 00000012004 15204243002 0015521 0 ustar 00 3 \� � @ s� d dl Z d dlZd dlZd dlZd dlZddlmZ ddlmZ ddlm Z dgZ ej dkrxe dg7 Z G d d� de �Z ne d g7 Z G d d � d e �ZG d d � d e �Ze� ZejZdS )� N� )�process)� reduction)�util�stopZwin32� DupSocketc @ s e Zd ZdZdd� Zdd� ZdS )r z Picklable wrapper for a socket.c s( |j � � � fdd�}tj|� j�| _d S )Nc s � j |�}| j|� d S )N)�shareZ send_bytes)�conn�pidr )�new_sock� �7/usr/lib64/python3.6/multiprocessing/resource_sharer.py�send s z DupSocket.__init__.<locals>.send)�dup�_resource_sharer�register�close�_id)�selfZsockr r )r r �__init__ s zDupSocket.__init__c C s* t j| j��}|j� }tj|�S Q R X dS )z1Get the socket. This should only be called once.N)r �get_connectionr Z recv_bytes�socketZ fromshare)r r r r r r �detach$ s zDupSocket.detachN)�__name__� __module__� __qualname__�__doc__r r r r r r r s �DupFdc @ s e Zd ZdZdd� Zdd� ZdS )r z-Wrapper for fd which can be used at any time.c s4 t j|�� � fdd�}� fdd�}tj||�| _d S )Nc s t j| � |� d S )N)r Z send_handle)r r )�new_fdr r r 1 s z DupFd.__init__.<locals>.sendc s t j� � d S )N)�osr r )r r r r 3 s z DupFd.__init__.<locals>.close)r r r r r )r �fdr r r )r r r / s zDupFd.__init__c C s" t j| j�� }tj|�S Q R X dS )z-Get the fd. This should only be called once.N)r r r r Z recv_handle)r r r r r r 7 s z DupFd.detachN)r r r r r r r r r r r - s c @ sN e Zd ZdZdd� Zdd� Zedd� �Zdd d �Zd d � Z d d� Z dd� Z dS )�_ResourceSharerz-Manager for resouces using background thread.c C s@ d| _ i | _g | _tj� | _d | _d | _d | _t j | t j � d S )Nr ) �_key�_cache� _old_locks� threading�Lock�_lock� _listener�_address�_threadr Zregister_after_forkr! � _afterfork)r r r r r ? s z_ResourceSharer.__init__c C sN | j �> | jdkr| j� | jd7 _||f| j| j< | j| jfS Q R X dS )z+Register resource, returning an identifier.Nr )r' r) �_startr" r# )r r r r r r r I s z_ResourceSharer.registerc C s< ddl m} | \}}||tj� jd�}|j|tj� f� |S )z<Return connection from which to receive identified resource.r )�Client)�authkey)� connectionr- r �current_processr. r r �getpid)Zidentr- �address�key�cr r r r R s z _ResourceSharer.get_connectionNc C s� ddl m} | j�� | jdk r�|| jtj� jd�}|jd� |j� | j j |� | j j � rdt j d� | jj� d| _ d| _d| _x | jj� D ]\}\}}|� q�W | jj� W dQ R X dS )z:Stop the background thread and clear registered resources.r )r- N)r. z._ResourceSharer thread did not stop when asked)r/ r- r' r) r r0 r. r r r* �joinZis_aliver Z sub_warningr( r# �items�clear)r Ztimeoutr- r4 r3 r r r r r r [ s" z_ResourceSharer.stopc C sn x | j j� D ]\}\}}|� q W | j j� | jj| j� tj� | _| jd k rX| jj � d | _d | _ d | _ d S )N) r# r6 r7 r$ �appendr' r% r& r( r r) r* )r r3 r r r r r r+ p s z_ResourceSharer._afterforkc C sX ddl m} tjd� |tj� jd�| _| jj| _ t j | j d�}d|_ |j� || _d S )Nr )�Listenerz0starting listener and thread for sending handles)r. )�targetT)r/ r9 r �debugr r0 r. r( r2 r) r% ZThread�_serveZdaemon�startr* )r r9 �tr r r r, ~ s z_ResourceSharer._startc C s� t td�r tjtjtdtj�� x�y\| jj� �H}|j� }|d krBP |\}}| j j |�\}}z|||� W d |� X W d Q R X W q" t j � s�t jt j� � Y q"X q"W d S )N�pthread_sigmaskr )�hasattr�signalr? � SIG_BLOCK�range�NSIGr( ZacceptZrecvr# �popr Z is_exiting�sys� excepthook�exc_info)r r �msgr3 Zdestination_pidr r r r r r< � s z_ResourceSharer._serve)N) r r r r r r � staticmethodr r r+ r, r< r r r r r! = s r! )r rA r rF r% � r �contextr r �__all__�platform�objectr r r! r r r r r r �<module> s ` __pycache__/popen_fork.cpython-36.opt-2.pyc 0000644 00000004051 15204243003 0014475 0 ustar 00 3 \ � @ s> d dl Z d dlZd dlZddlmZ dgZG dd� de�ZdS )� N� )�util�Popenc @ sH e Zd ZdZdd� Zdd� Zejfdd�Zdd d �Z d d � Z d d� Z dS )r �forkc C s t j� d | _| j|� d S )N)r Z_flush_std_streams� returncode�_launch)�self� process_obj� r �2/usr/lib64/python3.6/multiprocessing/popen_fork.py�__init__ s zPopen.__init__c C s |S )Nr )r �fdr r r �duplicate_for_child s zPopen.duplicate_for_childc C s� | j d krzx>ytj| j|�\}}W n tk rB } zd S d }~X q X P q W || jkrztj|�rntj|� | _ n tj|�| _ | j S )N)r �os�waitpid�pid�OSError� WIFSIGNALED�WTERMSIG� WEXITSTATUS)r �flagr �sts�er r r �poll s z Popen.pollNc C sN | j d krH|d k r0ddlm} || jg|�s0d S | j|dkrBtjnd�S | j S )Nr )�waitg )r Zmultiprocessing.connectionr �sentinelr r �WNOHANG)r �timeoutr r r r r + s z Popen.waitc C s\ | j d krXytj| jtj� W n8 tk r2 Y n& tk rV | jdd�d krR� Y nX d S )Ng�������?)r ) r r �killr �signal�SIGTERM�ProcessLookupErrorr r )r r r r � terminate5 s zPopen.terminatec C s� d}t j� \}}t j� | _| jdkrdz0t j|� dtjkrJdd l}|j� |j � }W d t j |� X n"t j|� t j | t j|f� || _ d S )Nr r �random)r �piper r �close�sys�modulesr# Zseed� _bootstrap�_exitr ZFinalizer )r r �codeZparent_rZchild_wr# r r r r ? s z Popen._launch)N) �__name__� __module__� __qualname__�methodr r r r r r r" r r r r r r s )r r& r � r �__all__�objectr r r r r �<module> s __pycache__/forkserver.cpython-36.pyc 0000644 00000015266 15204243003 0013575 0 ustar 00 3 \�! � @ s� d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZddlm Z ddlm Z ddl m Z ddlm Z ddlmZ ddlmZ d d d d gZd Zejd�ZG dd� de�Zddd�Zdd� Zdd� Zdd� Ze� ZejZej Z ej Z ej Z dS )� N� )� connection)�process)� reduction)�semaphore_tracker)�spawn)�util�ensure_running�get_inherited_fds�connect_to_new_process�set_forkserver_preload� �Qc @ s4 e Zd Zdd� Zdd� Zdd� Zdd� Zd d � Zd S ) � ForkServerc C s. d | _ d | _d | _d | _tj� | _dg| _d S )N�__main__)�_forkserver_address�_forkserver_alive_fd�_forkserver_pid�_inherited_fds� threadingZLock�_lock�_preload_modules)�self� r �2/usr/lib64/python3.6/multiprocessing/forkserver.py�__init__! s zForkServer.__init__c C s& t dd� | jD ��s td��|| _dS )z>Set list of module names to try to load in forkserver process.c s s | ]}t |�tkV qd S )N)�type�str)�.0�modr r r � <genexpr>+ s z4ForkServer.set_forkserver_preload.<locals>.<genexpr>z&module_names must be a list of stringsN)�allr � TypeError)r Z modules_namesr r r r ) s z!ForkServer.set_forkserver_preloadc C s | j S )z�Return list of fds inherited from parent process. This returns None if the current process was not started by fork server. )r )r r r r r / s z ForkServer.get_inherited_fdsc C s� | j � t|�d tkr td��tjtj���}|j| j� tj � \}}tj � \}}||| j t j � g}||7 }z<yt j||� ||fS tj|� tj|� � Y nX W dtj|� tj|� X W dQ R X dS )a; Request forkserver to create a child process. Returns a pair of fds (status_r, data_w). The calling process can read the child process's pid and (eventually) its returncode from status_r. The calling process should write to data_w the pickled preparation and process data. � z too many fdsN)r �len�MAXFDS_TO_SEND� ValueError�socket�AF_UNIXZconnectr �os�piper r Zgetfdr Zsendfds�close)r �fdsZclientZparent_r�child_w�child_rZparent_wZallfdsr r r r 7 s& z!ForkServer.connect_to_new_processc s� | j ��n tj� | jdk rVtj| jtj�\}}|s8dS tj| j� d| _ d| _d| _d }| j r�ddh� t j d�}t � fdd�|j� D ��}ni }tjtj���}tjd �}|j|� tj|d � |j� tj� \}}ztyV|j� |g} ||j� || j |f; }t j� } | gtj� } | d |g7 } tj| | | �}W n tj|� � Y nX W dtj|� X || _ || _|| _W dQ R X W dQ R X dS ) z�Make sure that a fork server is running. This can be called from any process. Note that usually a child process will just reuse the forkserver started by its parent, so ensure_running() will do nothing. Nz-from multiprocessing.forkserver import main; zmain(%d, %d, %r, **%r)� main_path�sys_path�ignorec 3 s" | ]\}}|� kr||fV qd S )Nr )r �x�y)� desired_keysr r r o s z,ForkServer.ensure_running.<locals>.<genexpr>r( i� z-czCfrom multiprocessing.forkserver import main; main(%d, %d, %r, **%r)) r r r r r) �waitpid�WNOHANGr+ r r r r Zget_preparation_data�dict�itemsr' r( r Zarbitrary_addressZbind�chmodZlistenr* �filenoZget_executabler Z _args_from_interpreter_flagsZspawnv_passfds) r �pidZstatus�cmd�data�listenerZaddress�alive_rZalive_wZ fds_to_passZexe�argsr )r4 r r T sL zForkServer.ensure_runningN)�__name__� __module__� __qualname__r r r r r r r r r r s r c 4 C s |rhd|kr8|dk r8dt j� _ztj|� W dt j� `X x.|D ]&}y t|� W q> tk rb Y q>X q>W tj� t j t j t j t j i}dd� |j � D �}tjtj| d���R}tj� ��<} |j� t_| j|tj� | j|tj� �x y�x dd� | j� D �} | r�P q�W || k�r$tj|d �d k�s t�t�|| k�s2t�|j � d �l} d } tj � d k�r�zFyt | |||� W n. t k �r� t j!t j"� � t j#j$� Y nX W dtj%| � X W dQ R X W q� t&k �r� } z| j't'j(k�rւ W Y dd} ~ X q�X q�W W dQ R X W dQ R X dS ) zRun forkserver.r NTc S s i | ]\}}t j ||�|�qS r )�signal)r �sig�valr r r � <dictcomp>� s zmain.<locals>.<dictcomp>)r: c S s g | ]\}}|j �qS r )Zfileobj)r �keyZeventsr r r � <listcomp>� s zmain.<locals>.<listcomp>r � r ))r Zcurrent_processZ _inheritingr Zimport_main_path� __import__� ImportErrorr Z _close_stdinrD �SIGCHLD�SIG_IGN�SIGINTr8 r' r( � selectorsZDefaultSelectorZ getsockname� _forkserverr �registerZ EVENT_READZselectr) �read�AssertionError� SystemExitZaccept�fork� _serve_one� Exception�sys� excepthook�exc_info�stderr�flush�_exit�OSError�errnoZ ECONNABORTED)Z listener_fdr? Zpreloadr/ r0 �modname�handlersZ old_handlersr> ZselectorZrfds�s�code�er r r �main� sV rf c C s� |j � tj |� x |j� D ]\}}tj||� q W tj| td �}| j � t|�tks\t�|^}}t _ } t _ | t j _t|tj� � dtjkr�dd l} | j� tj|�} t|| � d S )Nr �randomr )r+ r) r8 rD r Zrecvfdsr% r$ rT rQ r r r Z_semaphore_trackerZ_fd�write_unsigned�getpidrY �modulesrg Zseedr �_main) rc r> r? rb rE rF r, r. r- Zstfdrg rd r r r rW � s rW c C sR d}t j}x8t|�|k rBtj| |t|� �}|s8td��||7 }q W t j|�d S )NrJ zunexpected EOFr )�UNSIGNED_STRUCT�sizer$ r) rS �EOFErrorZunpack)�fdr= Zlengthrc r r r � read_unsigned� s rp c C s@ t j|�}x0|r:tj| |�}|dkr,td��||d � }q W d S )Nr zshould not get here)rl Zpackr) �write� RuntimeError)ro �n�msg�nbytesr r r rh � s rh )NN) r` r) rP rD r' ZstructrY r � r r �contextr r r r �__all__r% ZStructrl �objectr rf rW rp rh rQ r r r r r r r r �<module> s6 s =! __pycache__/synchronize.cpython-36.pyc 0000644 00000026460 15204243003 0013756 0 ustar 00 3 \/ � @ s, d dddddgZ ddlZddlZddlZddlZddlZdd lmZ dd lmZ dd lm Z ydd lm Z m Z W n e k r� e d ��Y nX e ed��\ZZej jZG dd� de�Z G dd� de �ZG dd� de�ZG dd � d e �ZG dd� de �ZG dd� de�ZG dd� de�ZG dd� dej�ZdS ) �Lock�RLock� Semaphore�BoundedSemaphore� Condition�Event� N� )�context)�process)�util)�SemLock� sem_unlinkz*This platform lacks a functioning sem_openz( implementation, therefore, the requiredz+ synchronization primitives needed will notz function, see issue 3770.� c @ s\ e Zd Zej� Zdd� Zedd� �Zdd� Z dd� Z d d � Z d d � Z d d� Z edd� �ZdS )r c C s� |d krt jj� }|j� }tjdkp*|dk}xPtd�D ]<}y tj|||| j � |� }| _ W n t k rn Y q6X P q6W t d��t j d|j � | j� tjdkr�dd� } t j| | � | j jd k r�dd lm} | | j j� t j| tj| j jfd d � d S ) N�win32�fork�d z cannot find name for semaphorez created semlock with handle %sc S s | j j� d S )N)�_semlock� _after_fork)�obj� r �3/usr/lib64/python3.6/multiprocessing/synchronize.pyr G s z%SemLock.__init__.<locals>._after_forkr )�registerr )Z exitpriority)r Z_default_contextZ get_contextZget_start_method�sys�platform�range�_multiprocessingr � _make_namer �FileExistsErrorr �debug�handle� _make_methodsZregister_after_fork�name�semaphore_trackerr ZFinalize�_cleanup) �self�kind�value�maxvalue�ctxr! Z unlink_now�i�slr r r r r �__init__2 s. zSemLock.__init__c C s ddl m} t| � || � d S )Nr )� unregister)r" r, r )r! r, r r r r# T s zSemLock._cleanupc C s | j j| _| j j| _d S )N)r �acquire�release)r$ r r r r Z s zSemLock._make_methodsc C s | j j� S )N)r � __enter__)r$ r r r r/ ^ s zSemLock.__enter__c G s | j j|� S )N)r �__exit__)r$ �argsr r r r0 a s zSemLock.__exit__c C sD t j| � | j}tjdkr,t j� j|j�}n|j}||j|j |j fS )Nr ) r �assert_spawningr r r Zget_spawning_popenZduplicate_for_childr r% r' r! )r$ r* �hr r r � __getstate__d s zSemLock.__getstate__c C s, t jj|� | _tjd|d � | j� d S )Nz recreated blocker with handle %rr )r r Z_rebuildr r r r )r$ �stater r r � __setstate__m s zSemLock.__setstate__c C s dt j� jd ttj�f S )Nz%s-%sZ semprefix)r �current_processZ_config�nextr �_randr r r r r r s zSemLock._make_nameN)�__name__� __module__� __qualname__�tempfileZ_RandomNameSequencer9 r+ � staticmethodr# r r/ r0 r4 r6 r r r r r r . s " r c @ s& e Zd Zd dd�Zdd� Zdd� ZdS ) r r c C s t j| t|t|d� d S )N)r( )r r+ � SEMAPHORE� SEM_VALUE_MAX)r$ r& r( r r r r+ } s zSemaphore.__init__c C s | j j� S )N)r � _get_value)r$ r r r � get_value� s zSemaphore.get_valuec C s8 y| j j� }W n tk r& d}Y nX d| jj|f S )N�unknownz<%s(value=%s)>)r rA � Exception� __class__r: )r$ r& r r r �__repr__� s zSemaphore.__repr__N)r )r: r; r< r+ rB rF r r r r r { s c @ s e Zd Zddd�Zdd� ZdS )r r c C s t j| t|||d� d S )N)r( )r r+ r? )r$ r& r( r r r r+ � s zBoundedSemaphore.__init__c C s> y| j j� }W n tk r& d}Y nX d| jj|| j jf S )NrC z<%s(value=%s, maxvalue=%s)>)r rA rD rE r: r' )r$ r& r r r rF � s zBoundedSemaphore.__repr__N)r )r: r; r< r+ rF r r r r r � s c @ s e Zd Zdd� Zdd� ZdS )r c C s t j| tdd|d� d S )Nr )r( )r r+ r? )r$ r( r r r r+ � s z Lock.__init__c C s� yf| j j� r8tj� j}tj� jdkrd|dtj� j 7 }n,| j j� dkrLd}n| j j� dkr`d}nd}W n t k r~ d}Y nX d | j j |f S ) N� MainThread�|r �Noner �SomeOtherThread�SomeOtherProcessrC z<%s(owner=%s)>) r �_is_miner r7 r! � threading�current_threadrA �_countrD rE r: )r$ r! r r r rF � s z Lock.__repr__N)r: r; r< r+ rF r r r r r � s c @ s e Zd Zdd� Zdd� ZdS )r c C s t j| tdd|d� d S )Nr )r( )r r+ �RECURSIVE_MUTEX)r$ r( r r r r+ � s zRLock.__init__c C s� y|| j j� rBtj� j}tj� jdkr6|dtj� j 7 }| j j� }n8| j j� dkrZd \}}n | j j� dkrrd \}}nd \}}W n t k r� d\}}Y nX d | j j ||f S )NrG rH r rI r rJ �nonzerorK rC z <%s(%s, %s)>)rI r )rJ rQ )rK rQ )rC rC ) r rL r r7 r! rM rN rO rA rD rE r: )r$ r! �countr r r rF � s zRLock.__repr__N)r: r; r< r+ rF r r r r r � s c @ sj e Zd Zddd�Zdd� Zdd� Zdd � Zd d � Zd d � Zdd� Z ddd�Z dd� Z dd� Z ddd�Z dS )r Nc C s> |p |j � | _|jd�| _|jd�| _|jd�| _| j� d S )Nr )r �_lockr �_sleeping_count� _woken_count�_wait_semaphorer )r$ �lockr( r r r r+ � s zCondition.__init__c C s t j| � | j| j| j| jfS )N)r r2 rS rT rU rV )r$ r r r r4 � s zCondition.__getstate__c C s |\| _ | _| _| _| j� d S )N)rS rT rU rV r )r$ r5 r r r r6 � s zCondition.__setstate__c C s | j j� S )N)rS r/ )r$ r r r r/ � s zCondition.__enter__c G s | j j|� S )N)rS r0 )r$ r1 r r r r0 � s zCondition.__exit__c C s | j j| _| j j| _d S )N)rS r- r. )r$ r r r r � s zCondition._make_methodsc C sJ y | j jj� | jjj� }W n tk r4 d}Y nX d| jj| j|f S )NrC z <%s(%s, %s)>)rT r rA rU rD rE r: rS )r$ Z num_waitersr r r rF � s zCondition.__repr__c C s� | j jj� std��| jj� | j jj� }xt|�D ]}| j j� q4W z| jj d|�S | j j� xt|�D ]}| j j � qjW X d S )Nz,must acquire() condition before using wait()T) rS r rL �AssertionErrorrT r. rO r rV r- rU )r$ �timeoutrR r) r r r �wait� s zCondition.waitc C s| | j jj� std��| jjd� s&t�x$| jjd�rJ| jjd�}|s(t�q(W | jjd�rx| jj� | jj� | jjd� d S )Nzlock is not ownedF) rS r rL rX rV r- rU rT r. )r$ �resr r r �notify s zCondition.notifyc C s� | j jj� std��| jjd� s&t�x$| jjd�rJ| jjd�}|s(t�q(W d}x"| jjd�rr| jj� |d7 }qRW |r�xt |�D ]}| jj� q�W x| jjd�r�q�W d S )Nzlock is not ownedFr r ) rS r rL rX rV r- rU rT r. r )r$ r[ Zsleepersr) r r r � notify_all s zCondition.notify_allc C sh |� }|r|S |d k r$t j� | }nd }d }x6|sb|d k rP|t j� }|dkrPP | j|� |� }q.W |S )Nr )�timeZ monotonicrZ )r$ Z predicaterY �resultZendtimeZwaittimer r r �wait_for6 s zCondition.wait_for)N)N)N)r: r; r< r+ r4 r6 r/ r0 r rF rZ r\ r] r` r r r r r � s c @ s6 e Zd Zdd� Zdd� Zdd� Zdd� Zd d d �Zd S ) r c C s |j |j� �| _|jd�| _d S )Nr )r r �_condr �_flag)r$ r( r r r r+ N s zEvent.__init__c C s0 | j � | jjd�r"| jj� dS dS Q R X d S )NFT)ra rb r- r. )r$ r r r �is_setR s z Event.is_setc C s6 | j �& | jjd� | jj� | j j� W d Q R X d S )NF)ra rb r- r. r] )r$ r r r �setY s z Event.setc C s" | j � | jjd� W d Q R X d S )NF)ra rb r- )r$ r r r �clear_ s z Event.clearNc C sT | j �D | jjd�r | jj� n | j j|� | jjd�rF| jj� dS dS Q R X d S )NFT)ra rb r- r. rZ )r$ rY r r r rZ c s z Event.wait)N)r: r; r< r+ rc rd re rZ r r r r r L s c @ sZ e Zd Zddd�Zdd� Zdd� Zedd � �Zejd d � �Zed d � �Z e jd d � �Z dS )�BarrierNc C sR dd l }ddlm} ||jd�d �}|j� }| j|||||f� d| _d| _d S )Nr r )� BufferWrapperr) r )�struct�heaprg Zcalcsizer r6 �_staterO ) r$ Zparties�actionrY r( rh rg �wrapperZcondr r r r+ u s zBarrier.__init__c C s. |\| _ | _| _| _| _| jj� jd�| _d S )Nr) )�_parties�_action�_timeoutra �_wrapperZcreate_memoryview�cast�_array)r$ r5 r r r r6 ~ s zBarrier.__setstate__c C s | j | j| j| j| jfS )N)rm rn ro ra rp )r$ r r r r4 � s zBarrier.__getstate__c C s | j d S )Nr )rr )r$ r r r rj � s zBarrier._statec C s || j d< d S )Nr )rr )r$ r& r r r rj � s c C s | j d S )Nr )rr )r$ r r r rO � s zBarrier._countc C s || j d< d S )Nr )rr )r$ r& r r r rO � s )NN) r: r; r< r+ r6 r4 �propertyrj �setterrO r r r r rf s s rf zRThis platform lacks a functioning sem_open implementation, therefore, the requiredz}This platform lacks a functioning sem_open implementation, therefore, the required synchronization primitives needed will notz�This platform lacks a functioning sem_open implementation, therefore, the required synchronization primitives needed will not function, see issue 3770.)�__all__rM r r= r r^ � r r r r r � ImportError�listr rP r? r@ �objectr r r r r r rf r r r r �<module> s0 Mz' __pycache__/popen_spawn_win32.cpython-36.opt-2.pyc 0000644 00000005122 15204243003 0015706 0 ustar 00 3 \� � @ s� d dl Z d dlZd dlZd dlZd dlZddlmZmZmZ ddl m Z ddl m Z dgZ dZ ejdkoreed d �Zejj� jd �ZG d d� de�ZdS ) � N� )� reduction�get_spawning_popen�set_spawning_popen)�spawn)�util�Popeni Zwin32�frozenFzpythonservice.exec @ s: e Zd ZdZdd� Zdd� Zd dd�Zd d � Zd d � ZdS )r r c C s t j|j�}tjd d�\}}tj|d�}t jtj � |d�}dj dd� |D ��}t |ddd���}y4tj t j � |d d d dd d d � \}} } } tj| � W n tj|� � Y nX | | _d | _|| _t|�| _tj| tj| jf� t| � z tj||� tj||� W d td � X W d Q R X d S ) Nr )Z parent_pidZ pipe_handle� c s s | ]}d | V qdS )z"%s"N� )�.0�xr r �9/usr/lib64/python3.6/multiprocessing/popen_spawn_win32.py� <genexpr>) s z!Popen.__init__.<locals>.<genexpr>�wbT)�closefdF)r Zget_preparation_data�_name�_winapiZ CreatePipe�msvcrtZopen_osfhandleZget_command_line�os�getpid�join�openZ CreateProcessZget_executableZ CloseHandle�pid� returncode�_handle�int�sentinelr ZFinalizer r �dump) �selfZ process_objZ prep_dataZrhandleZwhandleZwfd�cmdZto_childZhpZhtr �tidr r r �__init__ s2 zPopen.__init__c C s t j|| j�S )N)r Z duplicater )r Zhandler r r �duplicate_for_childE s zPopen.duplicate_for_childNc C st | j d krn|d krtj}ntdt|d d ��}tjt| j�|�}|tjkrntj| j�}|t krht j }|| _ | j S )Nr i� g �?) r r ZINFINITE�maxr ZWaitForSingleObjectr Z WAIT_OBJECT_0ZGetExitCodeProcess� TERMINATE�signal�SIGTERM)r �timeoutZmsecs�res�coder r r �waitI s z Popen.waitc C s | j dd�S )Nr )r( )r+ )r r r r �pollY s z Popen.pollc C sL | j d krHytjt| j�t� W n& tk rF | jdd�d krB� Y nX d S )Ng �?)r( )r r ZTerminateProcessr r r% �OSErrorr+ )r r r r � terminate\ s zPopen.terminate)N) �__name__� __module__� __qualname__�methodr" r# r+ r, r. r r r r r s % )r r r& �sysr �contextr r r � r r �__all__r% �platform�getattrZWINEXE� executable�lower�endswithZ WINSERVICE�objectr r r r r �<module> s __pycache__/spawn.cpython-36.opt-1.pyc 0000644 00000014401 15204243003 0013462 0 ustar 00 3 \�" � @ s. d dl Z d dlZd dlZd dlZddlmZmZ ddlmZ ddlm Z ddlm Z ddd d d d d gZ ej dkrzdZ dZn&ej dko�eedd�Z ejj� jd�Zer�e jjejd�anejadd � Zdd � Zdd� Zdd� Zdd � Zd&dd�Z dd� Z d d � Z d d � Z g Z d d � Z!d!d"� Z"d#d$� Z#d%d � Z$dS )'� N� )�get_start_method�set_start_method)�process)� reduction)�util�_main�freeze_support�set_executable�get_executable�get_preparation_data�get_command_line�import_main_path�win32F�frozenzpythonservice.exez python.exec C s | a d S )N)� _python_exe)Zexe� r �-/usr/lib64/python3.6/multiprocessing/spawn.pyr ) s c C s t S )N)r r r r r r - s c C s$ t | �dkr | d dkr dS dS dS )z= Return whether commandline indicates we are forking � r z--multiprocessing-forkTFN)�len)�argvr r r � is_forking4 s r c C sh t tj�rdi } xBtjdd� D ]0}|jd�\}}|dkrBd| |< q t|�| |< q W tf | � tj� dS )zE Run code for process object if this in not the main process r N�=�None)r �sysr �split�int� spawn_main�exit)�kwds�arg�name�valuer r r r > s c K sh t tdd�r(tjdgdd� | j� D � S d}|djdd � | j� D ��; }tj� }tg| d |dg S d S ) zJ Returns prefix of command line used for spawning a child process r Fz--multiprocessing-forkc S s g | ] }d | �qS )z%s=%rr )�.0�itemr r r � <listcomp>T s z$get_command_line.<locals>.<listcomp>z<from multiprocessing.spawn import spawn_main; spawn_main(%s)z, c s s | ]}d | V qdS )z%s=%rNr )r# r$ r r r � <genexpr>W s z#get_command_line.<locals>.<genexpr>z-cN)�getattrr � executable�items�joinr Z _args_from_interpreter_flagsr )r �progZoptsr r r r N s c C s\ t jdkr.ddl}tj|| �}|j|tj�}nddlm } ||j _ | }t |�}t j |� dS )z7 Run code specified by data received over pipe r r Nr )�semaphore_tracker)r �platform�msvcrtr Z steal_handleZopen_osfhandle�os�O_RDONLY� r, Z_semaphore_trackerZ_fdr r )Z pipe_handleZ parent_pidZ tracker_fdr. Z new_handle�fdr, Zexitcoder r r r \ s r c C s^ t j| ddd��@}dtj� _z$tjj|�}t|� tjj|�}W d tj� `X W d Q R X |j � S )N�rbT)�closefd) r/ �fdopenr �current_process� _inheritingr �pickle�load�prepare� _bootstrap)r2 Z from_parentZpreparation_data�selfr r r r m s c C s t tj� dd�rtd��d S )Nr7 Fa An attempt has been made to start a new process before the current process has finished its bootstrapping phase. This probably means that you are not using fork to start your child processes and you have forgotten to use the proper idiom in the main module: if __name__ == '__main__': freeze_support() ... The "freeze_support()" line can be omitted if the program is not going to be frozen to produce an executable.)r' r r6 � RuntimeErrorr r r r �_check_not_importing_mainy s r> c C s t � ttjtj� jd�}tjdk r2tjj� |d< t j j � }y|j d�}W n t k r^ Y n X tj||< |j| |t jtjtj� t� d� t jd }t|jdd�}|dk r�||d< nht jd ks�t o�t �r t|d d�}|dk �r tj j|� o�tjdk �r tj jtj|�}tj j |�|d < |S ) zM Return info about parent needed by child to unpickle process object )� log_to_stderr�authkeyN� log_levelr1 )r! �sys_path�sys_argv�orig_dir�dir� start_method�__main__r! �init_main_from_namer �__file__�init_main_from_path) r> �dictr Z_log_to_stderrr r6 r@ Z_loggerZgetEffectiveLevelr �path�copy�index� ValueError� ORIGINAL_DIR�updater r/ �getcwdr �modulesr' �__spec__r- �WINEXE� WINSERVICE�isabsr* �normpath)r! �drB �i� main_moduleZ main_mod_name� main_pathr r r r � s> c C s� d| kr| d t j� _d| kr,| d t j� _d| krD| d rDtj� d| kr^tj� j| d � d| krp| d t_ d| kr�| d t_ d| kr�t j | d � d| kr�| d t _ d | kr�t| d d d � d | kr�t| d � nd | kr�t| d � dS )zE Try to get current process ready to unpickle process object r! r@ r? rA rB rC rE rD rF T)ZforcerH rJ N)r r6 r! r@ r r? Z get_loggerZsetLevelr rL r r/ �chdirrP r �_fixup_main_from_name�_fixup_main_from_path)�datar r r r: � s, r: c C s~ t jd }| dks | jd�r d S t|jdd �| kr6d S tj|� tjd�}t j | ddd�}|j j |� | t jd< t jd< d S )NrG z .__main__r! � __mp_main__T)�run_nameZ alter_sys) r rS �endswithr' rT �old_main_modules�append�types� ModuleType�runpyZ run_module�__dict__rQ )Zmod_name� current_mainr[ � main_contentr r r r^ � s r^ c C s� t jd }tjjtjj| ��d }|dkr.d S t|dd �| krBd S tj|� t j d�}t j | dd�}|j j|� | t jd< t jd< d S )NrG r ZipythonrI ra )rb )r rS r/ rL �splitext�basenamer' rd re rf rg rh Zrun_pathri rQ )r\ rj Z main_namer[ rk r r r r_ � s r_ c C s t | � dS )z< Set sys.modules['__main__'] to module at main_path N)r_ )r\ r r r r s )NN)%r/ r rh rf r1 r r r �contextr r �__all__r- rU rV r' r( �lowerrc rL r* � exec_prefixr r r r r r r r r> r rd r: r^ r_ r r r r r �<module> s> 2& __pycache__/popen_spawn_posix.cpython-36.opt-2.pyc 0000644 00000004074 15204243003 0016113 0 ustar 00 3 \p � @ sp d dl Z d dlZddlmZmZ ddlmZ ddlmZ ddlmZ dgZ G dd � d e �Z G d d� dej �Z dS ) � N� )� reduction�set_spawning_popen)� popen_fork)�spawn)�util�Popenc @ s e Zd Zdd� Zdd� ZdS )�_DupFdc C s || _ d S )N)�fd)�selfr � r �9/usr/lib64/python3.6/multiprocessing/popen_spawn_posix.py�__init__ s z_DupFd.__init__c C s | j S )N)r )r r r r �detach s z _DupFd.detachN)�__name__� __module__� __qualname__r r r r r r r s r c s4 e Zd ZdZeZ� fdd�Zdd� Zdd� Z� Z S )r r c s g | _ t� j|� d S )N)�_fds�superr )r � process_obj)� __class__r r r s zPopen.__init__c C s | j j|� |S )N)r �append)r r r r r �duplicate_for_child" s zPopen.duplicate_for_childc C s@ ddl m} |j� }| jj|� tj|j�}tj � }t | � z t j ||� t j ||� W d t d � X d } } }} z~t j� \}}t j� \}} tj||d�} | jj||g� tjtj� | | j�| _|| _t| ddd��} | j|j� � W d Q R X W d |d k �rtj| t j|f� x(||| fD ]} | d k �r t j| � �q W X d S )Nr )�semaphore_tracker)� tracker_fdZ pipe_handle�wbF)�closefd)� r Zgetfdr r r Zget_preparation_data�_name�io�BytesIOr r �dump�os�pipeZget_command_line�extendr Zspawnv_passfdsZget_executable�pid�sentinel�open�write� getbufferZFinalize�close) r r r r Z prep_data�fpZparent_rZchild_wZchild_rZparent_w�cmd�fr r r r �_launch& s6 z Popen._launch) r r r �methodr ZDupFdr r r. � __classcell__r r )r r r s ) r r" �contextr r r r r r �__all__�objectr r r r r r �<module> s __pycache__/queues.cpython-36.opt-2.pyc 0000644 00000021242 15204243003 0013643 0 ustar 00 3 \ * � @ s� d ddgZ ddlZddlZddlZddlZddlZddlZddlZddlm Z m Z ddl Z ddl m Z ddl mZ ejjZdd lmZmZmZmZmZ G d d � d e�Ze� ZG d d� de�ZG d d� de�Z dS ) �Queue� SimpleQueue� JoinableQueue� N)�Empty�Full� )� connection)�context)�debug�info�Finalize�register_after_fork� is_exitingc @ s� e Zd Zd(dd�Zdd� Zdd� Zdd � Zd)d d �Zd*dd�Zdd� Z dd� Z dd� Z dd� Z dd� Z dd� Zd d � Zd d � Zd d!� Zed"d#� �Zed$d%� �Zed&d'� �Zd S )+r r c C s� |dkrddl m} || _tjdd�\| _| _|j� | _t j � | _ t j dkrTd | _n |j� | _|j|�| _d| _| j� t j dkr�t| tj� d S )Nr r )� SEM_VALUE_MAXF)�duplex�win32)Z synchronizer �_maxsizer �Pipe�_reader�_writer�Lock�_rlock�os�getpid�_opid�sys�platform�_wlockZBoundedSemaphore�_sem� _ignore_epipe� _after_forkr r )�self�maxsize�ctx� r$ �./usr/lib64/python3.6/multiprocessing/queues.py�__init__$ s zQueue.__init__c C s. t j| � | j| j| j| j| j| j| j| j fS )N) r �assert_spawningr r r r r r r r )r! r$ r$ r% � __getstate__9 s zQueue.__getstate__c C s0 |\| _ | _| _| _| _| _| _| _| j� d S )N) r r r r r r r r r )r! �stater$ r$ r% � __setstate__> s $zQueue.__setstate__c C sb t d� tjtj� �| _tj� | _d | _d | _ d| _ d| _ d | _ | j j| _| jj| _| jj| _d S )NzQueue._after_fork()F)r � threading� Conditionr � _notempty� collections�deque�_buffer�_thread� _jointhread�_joincancelled�_closed�_closer � send_bytes� _send_bytesr � recv_bytes� _recv_bytes�poll�_poll)r! r$ r$ r% r C s zQueue._after_forkTNc C sP | j j||�st�| j�. | jd kr,| j� | jj|� | jj� W d Q R X d S )N) r �acquirer r- r1 � _start_threadr0 �append�notify)r! �obj�block�timeoutr$ r$ r% �putP s z Queue.putc C s� |r2|d kr2| j � | j� }W d Q R X | jj� nr|rBtj� | }| j j||�sTt�zB|rv|tj� }| j|�s�t�n | j� s�t�| j� }| jj� W d | j j� X t j |�S )N) r r9 r �release�timeZ monotonicr< r r; �_ForkingPickler�loads)r! rA rB �resZdeadliner$ r$ r% �get[ s& z Queue.getc C s | j | jjj� S )N)r r �_semlockZ _get_value)r! r$ r$ r% �qsizes s z Queue.qsizec C s | j � S )N)r; )r! r$ r$ r% �emptyw s z Queue.emptyc C s | j jj� S )N)r rJ �_is_zero)r! r$ r$ r% �fullz s z Queue.fullc C s | j d�S )NF)rI )r! r$ r$ r% � get_nowait} s zQueue.get_nowaitc C s | j |d�S )NF)rC )r! r@ r$ r$ r% � put_nowait� s zQueue.put_nowaitc C s2 d| _ z| jj� W d | j}|r,d | _|� X d S )NT)r4 r �closer5 )r! rQ r$ r$ r% rQ � s z Queue.closec C s t d� | jr| j� d S )NzQueue.join_thread())r r2 )r! r$ r$ r% � join_thread� s zQueue.join_threadc C s6 t d� d| _y| jj� W n tk r0 Y nX d S )NzQueue.cancel_join_thread()T)r r3 r2 Zcancel�AttributeError)r! r$ r$ r% �cancel_join_thread� s zQueue.cancel_join_threadc C s� t d� | jj� tjtj| j| j| j| j | j j | j fdd�| _ d| j _t d� | j j� t d� | js�t| j tjtj| j �gd d�| _t| tj| j| jgd d�| _d S ) NzQueue._start_thread()ZQueueFeederThread)�target�args�nameTzdoing self._thread.start()z ... done self._thread.start()� )Z exitpriority� ���)r r0 �clearr+ ZThreadr �_feedr- r7 r r rQ r r1 Zdaemon�startr3 r �_finalize_join�weakref�refr2 �_finalize_closer5 )r! r$ r$ r% r= � s( zQueue._start_threadc C s4 t d� | � }|d k r(|j� t d� nt d� d S )Nzjoining queue threadz... queue thread joinedz ... queue thread already dead)r �join)ZtwrZthreadr$ r$ r% r^ � s zQueue._finalize_joinc C s. t d� |� | jt� |j� W d Q R X d S )Nz telling queue thread to quit)r r> � _sentinelr? )�buffer�notemptyr$ r$ r% ra � s zQueue._finalize_closec C sP t d� |j}|j}|j}| j} t} tjdkr<|j} |j} nd } �xy�|� z| sX|� W d |� X ybx\| � } | | kr�t d� |� d S tj | �} | d kr�|| � qh| � z || � W d | � X qhW W n t k r� Y nX W qD t k �rF } zJ|�rt |dd�t jk�rd S t� �r&td|� d S dd l}|j� W Y d d }~X qDX qDW d S )Nz$starting thread to feed data to piper z%feeder thread got sentinel -- exiting�errnor zerror in queue thread: %s)r r< rD �wait�popleftrc r r rF �dumps� IndexError� Exception�getattrrf ZEPIPEr r � traceback� print_exc)rd re r6 Z writelockrQ Z ignore_epipeZnacquireZnreleaseZnwaitZbpopleft�sentinelZwacquireZwreleaser@ �erm r$ r$ r% r\ � sR z Queue._feed)r )TN)TN)�__name__� __module__� __qualname__r& r( r* r rC rI rK rL rN rO rP rQ rR rT r= � staticmethodr^ ra r\ r$ r$ r$ r% r " s$ c @ s@ e Zd Zddd�Zdd� Zdd� Zdd d �Zd d � Zdd� Zd S )r r c C s* t j| ||d� |jd�| _|j� | _d S )N)r# r )r r& Z Semaphore�_unfinished_tasksr, �_cond)r! r" r# r$ r$ r% r& s zJoinableQueue.__init__c C s t j| �| j| jf S )N)r r( rv ru )r! r$ r$ r% r( s zJoinableQueue.__getstate__c C s, t j| |d d� � |dd � \| _| _d S )N� ���rx )r r* rv ru )r! r) r$ r$ r% r* s zJoinableQueue.__setstate__TNc C sl | j j||�st�| j�J | j�8 | jd kr4| j� | jj|� | j j � | jj � W d Q R X W d Q R X d S )N) r r< r r- rv r1 r= r0 r> ru rD r? )r! r@ rA rB r$ r$ r% rC s zJoinableQueue.putc C s@ | j �0 | jjd�s td��| jjj� r2| j j� W d Q R X d S )NFz!task_done() called too many times)rv ru r< � ValueErrorrJ rM Z notify_all)r! r$ r$ r% � task_done' s zJoinableQueue.task_donec C s, | j � | jjj� s | j j� W d Q R X d S )N)rv ru rJ rM rg )r! r$ r$ r% rb . s zJoinableQueue.join)r )TN) rq rr rs r& r( r* rC rz rb r$ r$ r$ r% r s c @ s< e Zd Zdd� Zdd� Zdd� Zdd� Zd d � Zd d � Zd S )r c C sH t jdd�\| _| _|j� | _| jj| _tj dkr:d | _ n |j� | _ d S )NF)r r ) r r r r r r r: r; r r r )r! r# r$ r$ r% r& 9 s zSimpleQueue.__init__c C s | j � S )N)r; )r! r$ r$ r% rL B s zSimpleQueue.emptyc C s t j| � | j| j| j| jfS )N)r r' r r r r )r! r$ r$ r% r( E s zSimpleQueue.__getstate__c C s" |\| _ | _| _| _| j j| _d S )N)r r r r r: r; )r! r) r$ r$ r% r* I s zSimpleQueue.__setstate__c C s& | j � | jj� }W d Q R X tj|�S )N)r r r8 rF rG )r! rH r$ r$ r% rI M s zSimpleQueue.getc C sD t j|�}| jd kr"| jj|� n | j� | jj|� W d Q R X d S )N)rF ri r r r6 )r! r@ r$ r$ r% rC S s zSimpleQueue.putN) rq rr rs r& rL r( r* rI rC r$ r$ r$ r% r 7 s ) �__all__r r r+ r. rE r_ rf Zqueuer r Z_multiprocessing� r r Z reductionZForkingPicklerrF �utilr r r r r �objectr rc r r r$ r$ r$ r% �<module> s$ b * __pycache__/heap.cpython-36.pyc 0000644 00000013677 15204243004 0012327 0 ustar 00 3 \ � @ s� d dl Z d dlZd dlZd dlZd dlZd dlZddlmZmZ ddl m Z dgZ ej dkrvd dl Z G dd� de�Zn,G d d� de�Zd d � Zd d � Zejee� G dd� de�ZG dd� de�ZdS )� N� )� reduction�assert_spawning)�util� BufferWrapperZwin32c @ s, e Zd Zej� Zdd� Zdd� Zdd� ZdS )�Arenac C sz || _ xTtd�D ]@}dtj� t| j�f }tjd||d�}tj� dkrHP |j � qW t d��|| _ || _ | j | j f| _ d S )N�d z pym-%d-%sr )�tagnamer z Cannot find name for new mmap���)�size�range�os�getpid�next�_rand�mmap�_winapiZ GetLastError�close�FileExistsError�name�buffer�_state)�selfr �ir Zbuf� r �,/usr/lib64/python3.6/multiprocessing/heap.py�__init__" s zArena.__init__c C s t | � | jS )N)r r )r r r r � __getstate__1 s zArena.__getstate__c C s, | \| _ | _| _tjd| j | jd�| _d S )Nr )r r )r r r r r )r �stater r r � __setstate__5 s zArena.__setstate__N) �__name__� __module__� __qualname__�tempfileZ_RandomNameSequencer r r r r r r r r s r c @ s e Zd Zddd�ZdS )r r c C s� || _ || _|d kr�tjdtj� tj� d�\| _}tj|� tj | tj | jf� t | jddd��^}d }||kr�d| }x t || �D ]}|j |� q�W ~|j d|| � |j� |ks�t�W d Q R X tj| j| j �| _d S ) Nr zpym-%d-)�prefix�dir�wbF)�closefdi � r i )r �fdr# Zmkstempr r r Z get_temp_dir�unlink�Finalizer �openr �write�tell�AssertionErrorr r )r r r) r �fZbsZzeros�_r r r r @ s" zArena.__init__Nr )r )r r! r" r r r r r r > s c C s( | j dkrtd��t| jtj| j �ffS )Nr zDArena is unpicklable because forking was enabled when it was createdr )r) � ValueError� rebuild_arenar r ZDupFd)�ar r r � reduce_arenaS s r5 c C s t | |j� �S )N)r �detach)r Zdupfdr r r r3 Y s r3 c @ sZ e Zd ZdZejfdd�Zedd� �Zdd� Z dd � Z d d � Z d d � Z dd� Z dd� ZdS )�Heap� c C sJ t j� | _tj� | _|| _g | _i | _i | _ i | _ t � | _ g | _ g | _d S )N)r r �_lastpid� threadingZLock�_lock�_size�_lengths� _len_to_seq�_start_to_block�_stop_to_block�set�_allocated_blocks�_arenas�_pending_free_blocks)r r r r r r f s z Heap.__init__c C s |d }| | | @ S )Nr r )�nZ alignment�maskr r r �_roundups s z Heap._roundupc C s� t j| j|�}|t| j�krj| jt| j|�tj�}| jd9 _t j d|� t |�}| j j |� |d|fS | j| }| j| }|j� }|s�| j|= | j|= |\}}}| j||f= | j||f= |S )N� z"allocating a new mmap of length %dr )�bisectZ bisect_leftr= �lenrG �maxr< r �PAGESIZEr �infor rC �appendr> �popr? r@ ) r r r �length�arena�seq�block�start�stopr r r �_mallocy s" z Heap._mallocc C s� |\}}}y| j ||f }W n tk r0 Y nX | j|�\}}y| j||f }W n tk rf Y nX | j|�\}}|||f}|| }y| j| j|� W n. tk r� |g| j|< tj| j|� Y nX || j||f< || j ||f< d S )N) r@ �KeyError�_absorbr? r> rN rI Zinsortr= ) r rS rQ rT rU Z prev_blockr1 Z next_blockrP r r r �_free� s( z Heap._freec C s^ |\}}}| j ||f= | j||f= || }| j| }|j|� |sV| j|= | jj|� ||fS )N)r? r@ r> �remover= )r rS rQ rT rU rP rR r r r rX � s z Heap._absorbc C sF x@y| j j� }W n tk r& P Y nX | jj|� | j|� qW d S )N)rD rO � IndexErrorrB rZ rY )r rS r r r �_free_pending_blocks� s zHeap._free_pending_blocksc C s` t j� | jkst�| jjd�s,| jj|� n0z"| j� | j j |� | j |� W d | jj � X d S )NF) r r r9 r/ r; �acquirerD rN r\ rB rZ rY �release)r rS r r r �free� s z Heap.freec C s� d| kot jk n s t�tj� | jkr4| j� | j�h | j� | j t |d�| j �}| j |�\}}}|| }||k r�| j |||f� |||f}| jj|� |S Q R X d S )Nr r )�sys�maxsizer/ r r r9 r r; r\ rG rK � _alignmentrV rY rB �add)r r rQ rT rU Znew_stoprS r r r �malloc� s z Heap.mallocN)r r! r" rb r rL r � staticmethodrG rV rY rX r\ r_ rd r r r r r7 b s r7 c @ s" e Zd Ze� Zdd� Zdd� ZdS )r c C sN d| kot jk n s t�tjj|�}||f| _tj| tjj |fd� d S )Nr )�args) r` ra r/ r �_heaprd r r r+ r_ )r r rS r r r r � s zBufferWrapper.__init__c C s& | j \\}}}}t|j�||| � S )N)r � memoryviewr )r rQ rT rU r r r r �create_memoryview� s z BufferWrapper.create_memoryviewN)r r! r" r7 rg r ri r r r r r � s )rI r r r` r# r: �contextr r � r �__all__�platformr �objectr r5 r3 �registerr7 r r r r r �<module> s$ __pycache__/reduction.cpython-36.opt-1.pyc 0000644 00000017350 15204243004 0014335 0 ustar 00 3 \ $ � @ s d dl mZmZ d dlZd dlZd dlZd dlZd dlZd dlZd dl Z ddl m Z ddddd gZ e j d kp�eed �o�eed �o�eejd �ZG dd� dej�ZejZd5dd �Ze j d k�re dddg7 Z d dlZd6dd�Zdd� Zdd� Zdd� ZG dd� de�ZnHe dddg7 Z d dlZe j d kZ d d� Z d d� Z d d� Zd d� Zd!d� Z d"d#� Z G d$d%� d%�Z!ee"e!� j#�e � d&d'� Z$ee"e%j&�e$� ee"e'j(�e$� d(d)� Z)d*d+� Z*eej+e)� e j d k�r�d,d-� Z,d.d/� Z-eeje,� n d0d-� Z,d1d/� Z-eeje,� G d2d3� d3ed4�Z.dS )7� )�ABCMeta�abstractmethodN� )�context� send_handle� recv_handle�ForkingPickler�register�dump�win32ZCMSG_LEN� SCM_RIGHTS�sendmsgc sJ e Zd ZdZi ZejZ� fdd�Ze dd� �Z e d dd��Z e j Z � ZS ) r z)Pickler subclass used by multiprocessing.c s* t � j|� | jj� | _| jj| j� d S )N)�super�__init__�_copyreg_dispatch_table�copy�dispatch_table�update�_extra_reducers)�self�args)� __class__� �1/usr/lib64/python3.6/multiprocessing/reduction.pyr & s zForkingPickler.__init__c C s || j |< dS )z&Register a reduce function for a type.N)r )�cls�type�reducer r r r + s zForkingPickler.registerNc C s t j� }| ||�j|� |j� S )N)�io�BytesIOr � getbuffer)r �obj�protocolZbufr r r �dumps0 s zForkingPickler.dumps)N)�__name__� __module__� __qualname__�__doc__r �copyregr r r � classmethodr r" �pickle�loads� __classcell__r r )r r r ! s c C s t ||�j| � dS )z3Replacement for pickle.dump() using ForkingPickler.N)r r )r �filer! r r r r : s � DupHandle� duplicate� steal_handleFc C s* |dkrt j� }t jt j� | |d|t j�S )z<Duplicate a handle. (target_process is a handle not a pid!)Nr )�_winapi�GetCurrentProcess�DuplicateHandle�DUPLICATE_SAME_ACCESS)�handleZtarget_processZ inheritabler r r r. G s c C sB t jt jd| �}z t j||t j� ddt jt jB �S t j|� X dS )z5Steal a handle from process identified by source_pid.Fr N)r0 � OpenProcess�PROCESS_DUP_HANDLEr2 r1 r3 �DUPLICATE_CLOSE_SOURCE� CloseHandle)Z source_pidr4 Zsource_process_handler r r r/ O s c C s t |tj|�}| j|� dS )z&Send a handle over a local connection.N)r- r0 r3 �send)�connr4 �destination_pidZdhr r r r [ s c C s | j � j� S )z)Receive a handle over a local connection.)�recv�detach)r: r r r r ` s c @ s" e Zd ZdZddd�Zdd� ZdS )r- z Picklable wrapper for a handle.Nc C s\ |d krt j� }tjtjd|�}z tjtj� |||dd�| _W d tj|� X || _ || _ d S )NFr ) �os�getpidr0 r5 r6 r2 r1 �_handler8 �_access�_pid)r r4 �access�pid�procr r r r f s zDupHandle.__init__c C sV | j tj� kr| jS tjtjd| j �}z tj|| jtj� | j dtj �S tj |� X dS )z1Get the handle. This should only be called once.FN) rB r> r? r@ r0 r5 r6 r2 r1 rA r7 r8 )r rE r r r r= u s zDupHandle.detach)N)r# r$ r% r& r r= r r r r r- d s �DupFd�sendfds�recvfds�darwinc C sV t j d|�}tt|�d g�}| j|gtjtj|fg� trR| jd�dkrRt d��dS )z,Send an array of fds over an AF_UNIX socket.�i� r � Az%did not receive acknowledgement of fdN) �array�bytes�lenr �socket� SOL_SOCKETr � ACKNOWLEDGEr< � RuntimeError)�sockZfds�msgr r r rG � s c C s� t j d�}|j| }| jdtj|��\}}}}| r>| r>t�yxtrN| jd� t|�dkrjt dt|� ��|d \}} } |tj kr�| tj kr�t| �|j dkr�t �|j | � t|�S W n t tfk r� Y nX t d��dS )z/Receive an array of fds over an AF_UNIX socket.rJ r rL z received %d items of ancdatar zInvalid data receivedN)rM �itemsizeZrecvmsgrP Z CMSG_SPACE�EOFErrorrR r9 rO rS rQ r � ValueErrorZ frombytes�list� IndexError) rT �size�aZ bytes_sizerU Zancdata�flagsZaddrZ cmsg_levelZ cmsg_typeZ cmsg_datar r r rH � s* c C s2 t j| j� t jt j��}t||g� W dQ R X dS )z&Send a handle over a local connection.N)rP �fromfd�fileno�AF_UNIX� SOCK_STREAMrG )r: r4 r; �sr r r r � s c C s0 t j| j� t jt j��}t|d�d S Q R X dS )z)Receive a handle over a local connection.r r N)rP r^ r_ r` ra rH )r: rb r r r r � s c C sF t j� }|dk r |j|j| ��S tr:ddlm} |j| �S td��dS )zReturn a wrapper for an fd.Nr )�resource_sharerz&SCM_RIGHTS appears not to be available)r Zget_spawning_popenrF Zduplicate_for_child�HAVE_SEND_HANDLE� rc rX )�fdZ popen_objrc r r r rF � s c C s2 | j d kr t| j| jjffS t| j | jjffS d S )N)�__self__�getattrr �__func__r# )�mr r r �_reduce_method� s rk c @ s e Zd Zdd� ZdS )�_Cc C s d S )Nr )r r r r �f� s z_C.fN)r# r$ r% rm r r r r rl � s rl c C s t | j| jffS )N)rh � __objclass__r# )rj r r r �_reduce_method_descriptor� s ro c C s t | j| j| jpi ffS )N)�_rebuild_partial�funcr �keywords)�pr r r �_reduce_partial� s rt c C s t j| f|�|�S )N)� functools�partial)rq r rr r r r rp � s rp c C s ddl m} t|| �ffS )Nr )� DupSocket)rc rw �_rebuild_socket)rb rw r r r �_reduce_socket� s ry c C s | j � S )N)r= )Zdsr r r rx � s rx c C s" t | j� �}t|| j| j| jffS )N)rF r_ rx �familyr �proto)rb �dfr r r ry � s c C s | j � }tj||||d�S )N)r_ )r= rP )r| rz r r{ rf r r r rx � s c @ sd e Zd ZdZeZeZeZeZeZe j dkr8e Z e Z e Z n eZeZeZeZeZeZeZeZdd� ZdS )�AbstractReducerz�Abstract base class for use in implementing a Reduction class suitable for use in replacing the standard reduction mechanism used in multiprocessing.r c G sN t tt� j�t� t ttj�t� t ttj �t� t t j t � t t j t� d S )N)r r rl rm rk rY �appendro �int�__add__ru rv rt rP ry )r r r r r r s zAbstractReducer.__init__N)r# r$ r% r& r r r r r �sys�platformr/ r. r- rG rH rF rk ro rp ry rx r r r r r r} � s& r} )� metaclass)N)NF)/�abcr r r' ru r r> r) rP r� re r �__all__r� �hasattrrd ZPicklerr r r r0 r. r/ r r �objectr- rM rR rG rH rF rk rl r rm ro rY r~ r r� rt rp rv ry rx r} r r r r �<module> sb # __pycache__/semaphore_tracker.cpython-36.opt-2.pyc 0000644 00000006403 15204243004 0016035 0 ustar 00 3 \ � @ s� d dl Z d dlZd dlZd dlZd dlZd dlZddlmZ ddlmZ dddgZ G dd � d e �Z e � Z e j Z e jZe jZe jZd d � ZdS ) � N� )�spawn)�util�ensure_running�register� unregisterc @ s< e Zd Zdd� Zdd� Zdd� Zdd� Zd d � Zd d � Zd S )�SemaphoreTrackerc C s t j� | _d | _d | _d S )N)� threadingZLock�_lock�_fd�_pid)�self� r �9/usr/lib64/python3.6/multiprocessing/semaphore_tracker.py�__init__ s zSemaphoreTracker.__init__c C s | j � | jS )N)r r )r r r r �getfd" s zSemaphoreTracker.getfdc C s | j �� | jd k rRtj| jtj�\}}|s0d S tj| j� d | _d | _tjd� g }y|j t j j � � W n t k r~ Y nX d}tj� \}}zlyB|j |� tj� }|gtj� }|d|| g7 }tj|||�}W n tj|� � Y nX || _|| _W d tj|� X W d Q R X d S )NzWsemaphore_tracker: process died unexpectedly, relaunching. Some semaphores might leak.z;from multiprocessing.semaphore_tracker import main;main(%d)z-c)r r �os�waitpid�WNOHANG�closer �warnings�warn�append�sys�stderr�fileno� Exception�piper Zget_executabler Z _args_from_interpreter_flagsZspawnv_passfds) r �pidZstatusZ fds_to_pass�cmd�r�wZexe�argsr r r r & s8 z SemaphoreTracker.ensure_runningc C s | j d|� d S )NZREGISTER)�_send)r �namer r r r Q s zSemaphoreTracker.registerc C s | j d|� d S )NZ UNREGISTER)r# )r r$ r r r r U s zSemaphoreTracker.unregisterc C s@ | j � dj||�jd�}t|�dkr.td��tj| j|�}d S )Nz{0}:{1} �asciii z name too long)r �format�encode�len� ValueErrorr �writer )r r r$ �msg�nbytesr r r r# Y s zSemaphoreTracker._sendN) �__name__� __module__� __qualname__r r r r r r# r r r r r s +r c - C s� t j t jt j� t j t jt j� x6tjtjfD ]&}y |j� W q. tk rR Y q.X q.W t � }z�t | d���}x�|D ]�}yJ|j � j d�\}}|dkr�|j |� n |dkr�|j|� n td| ��W qr tk r� ytjtj� � W n Y nX Y qrX qrW W d Q R X W d |�r<ytjdt|� � W n tk �r: Y nX xf|D ]^}zT|jd�}ytj|� W n6 tk �r� } ztjd||f � W Y d d }~X nX W d X �qBW X d S ) N�rb� :s REGISTERs UNREGISTERzunrecognized command %rzRsemaphore_tracker: There appear to be %d leaked semaphores to clean up at shutdownr% zsemaphore_tracker: %r: %s)�signal�SIGINT�SIG_IGN�SIGTERMr �stdin�stdoutr r �set�open�strip�split�add�remove� RuntimeError� excepthook�exc_infor r r( �decode�_multiprocessingZ sem_unlink)�fd�f�cache�liner r$ �er r r �maink sJ (rH )r r2 r r r rB � r r �__all__�objectr Z_semaphore_trackerr r r r rH r r r r �<module> s I __pycache__/popen_spawn_posix.cpython-36.pyc 0000644 00000004074 15204243004 0015154 0 ustar 00 3 \p � @ sp d dl Z d dlZddlmZmZ ddlmZ ddlmZ ddlmZ dgZ G dd � d e �Z G d d� dej �Z dS ) � N� )� reduction�set_spawning_popen)� popen_fork)�spawn)�util�Popenc @ s e Zd Zdd� Zdd� ZdS )�_DupFdc C s || _ d S )N)�fd)�selfr � r �9/usr/lib64/python3.6/multiprocessing/popen_spawn_posix.py�__init__ s z_DupFd.__init__c C s | j S )N)r )r r r r �detach s z _DupFd.detachN)�__name__� __module__� __qualname__r r r r r r r s r c s4 e Zd ZdZeZ� fdd�Zdd� Zdd� Z� Z S )r r c s g | _ t� j|� d S )N)�_fds�superr )r � process_obj)� __class__r r r s zPopen.__init__c C s | j j|� |S )N)r �append)r r r r r �duplicate_for_child" s zPopen.duplicate_for_childc C s@ ddl m} |j� }| jj|� tj|j�}tj � }t | � z t j ||� t j ||� W d t d � X d } } }} z~t j� \}}t j� \}} tj||d�} | jj||g� tjtj� | | j�| _|| _t| ddd��} | j|j� � W d Q R X W d |d k �rtj| t j|f� x(||| fD ]} | d k �r t j| � �q W X d S )Nr )�semaphore_tracker)� tracker_fdZ pipe_handle�wbF)�closefd)� r Zgetfdr r r Zget_preparation_data�_name�io�BytesIOr r �dump�os�pipeZget_command_line�extendr Zspawnv_passfdsZget_executable�pid�sentinel�open�write� getbufferZFinalize�close) r r r r Z prep_data�fpZparent_rZchild_wZchild_rZparent_w�cmd�fr r r r �_launch& s6 z Popen._launch) r r r �methodr ZDupFdr r r. � __classcell__r r )r r r s ) r r" �contextr r r r r r �__all__�objectr r r r r r �<module> s __pycache__/managers.cpython-36.opt-2.pyc 0000644 00000073242 15204243004 0014141 0 ustar 00 3 \� � @ s� d dddgZ ddlZddlZddlZddlZddlZddlmZ ddlm Z dd l m Z m Z dd lm Z dd lmZ dd lmZ dd lmZ dd� Ze jeje� dd� d�D �Zed ek r�dd� ZxeD ]Ze jee� q�W G dd� de�Zf i fdd�Zdd� ZG d d � d e�Z d d � Z d d!� Z G d"d#� d#e�Z G d$d%� d%e�Z e j!e j"fe j#e j$fd&�Z%G d'd � d e�Z&G d(d)� d)e'�Z(G d*d� de�Z)d+d,� Z*i fd-d.�Z+d�d0d1�Z,G d2d3� d3e�Z-G d4d5� d5e�Z.d�d6d7�Z/G d8d9� d9e)�Z0G d:d;� d;e)�Z1G d<d=� d=e1�Z2G d>d?� d?e)�Z3G d@dA� dAe)�Z4G dBdC� dCe)�Z5G dDdE� dEe)�Z6e+dFd��Z7G dZd[� d[e7�Z8e+d\d��Z9d]deie9_:e+dfd��Z;e+dgd��Z<dsdsdsdededt�e<_:G dudg� dge<�Z=G dvd� de&�Z>e>jdwej?� e>jdxej?� e>jdyej@e3� e>jdzejAe1� e>jd{ejBe1� e>jd|ejCe1� e>jd}ejDe1� e>jd~ejEe2� e>jdejFe4� e>jd�e jGe=� e>jd�ee8� e>jd�eHe9� e>jd5e.e6� e>jd7e/e;� e>jd3e-e5� e>jdee0d�d�� e>jdsd�d�� dS )�� BaseManager� SyncManager� BaseProxy�Token� N)� format_exc� )� connection)� reduction�get_spawning_popen)�pool)�process)�util)� get_contextc C s t j | j| j� ffS )N)�array�typecode�tobytes)�a� r �0/usr/lib64/python3.6/multiprocessing/managers.py� reduce_array$ s r c C s g | ]}t ti |�� ��qS r )�type�getattr)�.0�namer r r � <listcomp>( s r �items�keys�valuesc C s t t | �ffS )N)�list)�objr r r �rebuild_as_list* s r c @ s0 e Zd Zd Zdd� Zdd� Zdd � Zd d � Zd S )r �typeid�address�idc C s ||| | _ | _| _d S )N)r! r" r# )�selfr! r" r# r r r �__init__9 s zToken.__init__c C s | j | j| jfS )N)r! r" r# )r$ r r r � __getstate__< s zToken.__getstate__c C s |\| _ | _| _d S )N)r! r" r# )r$ �stater r r � __setstate__? s zToken.__setstate__c C s d| j j| j| j| jf S )Nz %s(typeid=%r, address=%r, id=%r))� __class__�__name__r! r" r# )r$ r r r �__repr__B s zToken.__repr__N)r! r" r# )r* � __module__� __qualname__� __slots__r% r& r( r+ r r r r r 3 s c C s8 | j ||||f� | j� \}}|dkr*|S t||��d S )Nz#RETURN)�send�recv�convert_to_error)�cr# � methodname�args�kwds�kind�resultr r r �dispatchJ s r8 c C s< | dkr |S | dkr t |�S | dkr0t d| �S td�S d S )Nz#ERRORz #TRACEBACKz#UNSERIALIZABLEzUnserializable message: %s zUnrecognized message type)� RemoteError� ValueError)r6 r7 r r r r1 T s r1 c @ s e Zd Zdd� ZdS )r9 c C s& ddd d t | jd � dd S )N� �-�K r )�strr4 )r$ r r r �__str__a s zRemoteError.__str__N)r* r, r- r? r r r r r9 ` s r9 c C s6 g }x,t | �D ] }t| |�}t|�r|j|� qW |S )N)�dirr �callable�append)r �tempr �funcr r r � all_methodsh s rE c C s dd� t | �D �S )Nc S s g | ]}|d dkr|�qS )r �_r )r r r r r r w s z"public_methods.<locals>.<listcomp>)rE )r r r r �public_methodss s rG c @ s� e Zd Zddddddddd g Zd d � Zd d � Zdd� Zdd� Zdd� Zdd� Z dd� Z dd� Z e e e d�Z dd � Z d d � Zd d � Zd!d"� Zd#d$� Zd%d&� Zd'd(� Zd)d*� Zd+d,� Zd-S ).�Server�shutdown�create�accept_connection� get_methods� debug_info�number_of_objects�dummy�incref�decrefc C s^ || _ tj|�| _t| \}}||dd�| _| jj| _dd f fi| _i | _i | _ t j � | _ d S )N� )r" Zbacklog�0) �registryr �AuthenticationString�authkey�listener_client�listenerr" � id_to_obj�id_to_refcount�id_to_local_proxy_obj� threading�Lock�mutex)r$ rT r" rV � serializer�Listener�Clientr r r r% � s zServer.__init__c C s� t j� | _| tj� _zZt j| jd�}d|_|j � y x| jj � sN| jj d� q6W W n t t fk rj Y nX W d tjtjkr�tjd� tjt_tjt_tjd� X d S )N)�targetTr zresetting stdout, stderrr )r\ �Event� stop_eventr �current_process�_manager_server�Thread�accepter�daemon�start�is_set�wait�KeyboardInterrupt� SystemExit�sys�stdout� __stdout__r �debug� __stderr__�stderr�exit)r$ rh r r r � serve_forever� s zServer.serve_foreverc C sP xJy| j j� }W n tk r& wY nX tj| j|fd�}d|_|j� qW d S )N)rb r4 T)rX Zaccept�OSErrorr\ rg �handle_requestri rj )r$ r2 �tr r r rh � s zServer.accepterc ' C s4 d } }}y>t j|| j� t j|| j� |j� }|\}}}}t| |�}W n tk rh dt� f} Y n>X y||f|�|�}W n tk r� dt� f} Y n X d|f} y|j| � W nr tk �r& } zTy|jdt� f� W n tk r� Y nX t j d| � t j d|� t j d| � W Y d d } ~ X nX |j � d S )Nz #TRACEBACKz#RETURNzFailure to send message: %rz ... request was %rz ... exception was %r) r Zdeliver_challengerV Zanswer_challenger0 r � Exceptionr r/ r �info�close) r$ r2 �funcnamer7 �request�ignorer4 r5 rD �msg�er r r rx � s2 zServer.handle_requestc D C s� t jdtj� j� |j}|j}| j}�x�| jj � �s�yBd }}|� }|\}}} } y|| \}} } W n^ t k r� } zBy| j | \}} } W n& t k r� } z | �W Y d d }~X nX W Y d d } ~ X nX || kr�t d|t |�| f ��t||�}y|| | �}W n, tk �r& } zd|f}W Y d d }~X nPX | �o8| j|d �}|�rn| j|||�\}}t|| j|�}d||ff}nd|f}W n� t k �r� |d k�r�dt� f}nNy,| j| }|| |||f| �| �}d|f}W n tk �r� dt� f}Y nX Y nP tk �r" t jdtj� j� tjd� Y n tk �r@ dt� f}Y nX yDy ||� W n2 tk �r� } z|d t� f� W Y d d }~X nX W q( tk �r� } zBt jd tj� j� t jd |� t jd |� |j� tjd � W Y d d }~X q(X q(W d S )Nz$starting server thread to service %rz+method %r of %r object is not in exposed=%rz#ERRORz#PROXYz#RETURNz #TRACEBACKz$got EOF -- exiting thread serving %rr z#UNSERIALIZABLEz exception in thread serving %rz ... message was %rz ... exception was %rr )r rr r\ �current_threadr r0 r/ rY rd rk �KeyErrorr[ �AttributeErrorr r rz �getrJ r r" r �fallback_mapping�EOFErrorro ru r{ r| )r$ �connr0 r/ rY r3 r r~ �identr4 r5 �exposed� gettypeid�keZ second_keZfunction�resr� r� r! ZridentZrexposed�tokenZ fallback_funcr7 r r r � serve_client� sx ( $ zServer.serve_clientc C s |S )Nr )r$ r� r� r r r r �fallback_getvalue& s zServer.fallback_getvaluec C s t |�S )N)r> )r$ r� r� r r r r � fallback_str) s zServer.fallback_strc C s t |�S )N)�repr)r$ r� r� r r r r � fallback_repr, s zServer.fallback_repr)r? r+ z #GETVALUEc C s d S )Nr )r$ r2 r r r rO 5 s z Server.dummyc C s| | j �l g }t| jj� �}|j� xD|D ]<}|dkr(|jd|| j| t| j| d �d d� f � q(W dj|�S Q R X d S )NrS z %s: refcount=%s %sr r= r; ) r^ r rZ r �sortrB r> rY �join)r$ r2 r7 r r� r r r rM 8 s $zServer.debug_infoc C s t | j�S )N)�lenrZ )r$ r2 r r r rN G s zServer.number_of_objectsc C sL z:yt jd� |jd� W n dd l}|j� Y nX W d | jj� X d S )Nz!manager received shutdown message�#RETURNr )r� N)r rr r/ � traceback� print_excrd �set)r$ r2 r� r r r rI N s zServer.shutdownc O s� | j �� | j| \}}}}|d kr,|d } n |||�} |d krFt| �}|d k r^t|�t|� }dt| � } tjd|| � | t|�|f| j| < | | j kr�d| j | < W d Q R X | j || � | t |�fS )Nr z%xz&%r callable returned object with id %r) r^ rT rG r r# r rr r� rY rZ rP �tuple) r$ r2 r! r4 r5 rA r� �method_to_typeid� proxytyper r� r r r rJ [ s z Server.createc C s t | j|j d �S )Nr )r� rY r# )r$ r2 r� r r r rL z s zServer.get_methodsc C s" |t j� _|jd� | j|� d S )N�#RETURN)r� N)r\ r� r r/ r� )r$ r2 r r r r rK � s zServer.accept_connectionc C s� | j �� y| j| d7 < W nh tk r� } zL|| jkrrd| j|< | j| | j|< | j| \}}}tjd|� n|�W Y d d }~X nX W d Q R X d S )Nr z&Server re-enabled tracking & INCREF %r)r^ rZ r� r[ rY r rr )r$ r2 r� r� r r� r� r r r rP � s z Server.increfc C s� || j kr$|| jkr$tjd|� d S | j�. | j | d8 < | j | dkrT| j |= W d Q R X || j kr�d f d f| j|< tjd|� | j� | j|= W d Q R X d S )NzServer DECREF skipping %rr r zdisposing of obj with id %r)rZ r[ r rr r^ rY )r$ r2 r� r r r rQ � s z Server.decrefN)r* r, r- Zpublicr% rv rh rx r� r� r� r� r� rO rM rN rI rJ rL rK rP rQ r r r r rH } s, "Q rH c @ s e Zd ZdgZdZdZdZdS )�State�valuer r � N)r* r, r- r. �INITIAL�STARTED�SHUTDOWNr r r r r� � s r� )�pickleZ xmlrpclibc @ s� e Zd Zi ZeZd dd�Zdd� Zdd� Zdf fd d �Z e df fd d ��Z d d� Z d!dd�Z dd� Zdd� Zdd� Zdd� Zedd� �Zedd � �Ze d"d d ��ZdS )#r Nr� c C s\ |d krt j� j}|| _t j|�| _t� | _tj| j_ || _ t | \| _ | _ |pTt� | _d S )N)r re rV �_addressrU �_authkeyr� �_stater� r� � _serializerrW Z _Listener�_Clientr �_ctx)r$ r" rV r_ Zctxr r r r% � s zBaseManager.__init__c C s t | j| j| j| j�S )N)rH � _registryr� r� r� )r$ r r r � get_server� s zBaseManager.get_serverc C s8 t | j \}}|| j| jd�}t|d d� tj| j_d S )N)rV rO ) rW r� r� r� r8 r� r� r� r� )r$ r` ra r� r r r �connect� s zBaseManager.connectc C s� |d k rt |� rtd��tjdd�\}}| jjt| �j| j| j | j | j |||fd�| _ dj dd� | j jD ��}t| �jd | | j _| j j� |j� |j� | _ |j� tj| j_tj| t| �j| j | j | j | j| jfd d �| _ d S ) Nz initializer must be a callableF)Zduplex)rb r4 �:c s s | ]}t |�V qd S )N)r> )r �ir r r � <genexpr>� s z$BaseManager.start.<locals>.<genexpr>r< r )r4 � exitpriority) rA � TypeErrorr ZPiper� ZProcessr � _run_serverr� r� r� r� �_processr� Z _identityr* r rj r| r0 r� r� r� r� r �Finalize�_finalize_managerr� rI )r$ � initializer�initargs�reader�writerr� r r r rj � s& zBaseManager.startc C sN |d k r||� | j ||||�}|j|j� |j� tjd|j� |j� d S )Nzmanager serving at %r)�_Serverr/ r" r| r r{ rv ) �clsrT r" rV r_ r� r� r� �serverr r r r� s zBaseManager._run_serverc O sN | j | j| jd�}z t|d d|f| |�\}}W d |j� X t|| j|�|fS )N)rV rJ )r� r� r� r8 r| r )r$ r! r4 r5 r� r# r� r r r �_create% s zBaseManager._createc C s* | j d k r&| j j|� | j j� s&d | _ d S )N)r� r� �is_alive)r$ �timeoutr r r r� 1 s zBaseManager.joinc C s. | j | j| jd�}z t|d d�S |j� X d S )N)rV rM )r� r� r� r8 r| )r$ r� r r r � _debug_info: s zBaseManager._debug_infoc C s. | j | j| jd�}z t|d d�S |j� X d S )N)rV rN )r� r� r� r8 r| )r$ r� r r r �_number_of_objectsD s z BaseManager._number_of_objectsc C s | j jtjkr| j� | S )N)r� r� r� r� rj )r$ r r r � __enter__N s zBaseManager.__enter__c C s | j � d S )N)rI )r$ �exc_type�exc_val�exc_tbr r r �__exit__T s zBaseManager.__exit__c C s� | j � r�tjd� y,|||d�}zt|d d� W d |j� X W n tk rR Y nX | jdd� | j � r�tjd� t| d�r�tjd� | j� | jd d� | j � r�tjd � t j |_ y t j |= W n tk r� Y nX d S ) Nz#sending shutdown message to manager)rV rI g �?)r� zmanager still alive� terminatez'trying to `terminate()` manager processg�������?z#manager still alive after terminate)r� r r{ r8 r| rz r� �hasattrr� r� r� r� r �_address_to_localr� )r r" rV r' r� r� r r r r� W s. z BaseManager._finalize_managerc C s | j S )N)r� )r$ r r r �<lambda>w s zBaseManager.<lambda>Tc s� d| j kr| jj� | _� d kr"t� |p0t� dd �}|p@t� dd �}|r`xt|j� �D ]\}}qTW |||� f| j�< |r�� �fdd�} �| _t| �| � d S )Nr� � _exposed_�_method_to_typeid_c s` t jd�� | j�f|�|�\}}� || j| | j|d�}| j|j| jd�}t|d d|jf� |S )Nz)requesting creation of a shared %r object)�managerrV r� )rV rQ ) r rr r� r� r� r� r" r8 r# )r$ r4 r5 r� Zexp�proxyr� )r� r! r r rC � s z"BaseManager.register.<locals>.temp) �__dict__r� �copy� AutoProxyr r r r* �setattr) r� r! rA r� r� r� � create_method�keyr� rC r )r� r! r �registery s zBaseManager.register)NNr� N)N)NNNNT)r* r, r- r� rH r� r% r� r� rj � classmethodr� r� r� r� r� r� r� � staticmethodr� �propertyr" r� r r r r r � s( $ c @ s e Zd Zdd� Zdd� ZdS )�ProcessLocalSetc C s t j| dd� � d S )Nc S s | j � S )N)�clear)r r r r r� � s z*ProcessLocalSet.__init__.<locals>.<lambda>)r �register_after_fork)r$ r r r r% � s zProcessLocalSet.__init__c C s t | �f fS )N)r )r$ r r r � __reduce__� s zProcessLocalSet.__reduce__N)r* r, r- r% r� r r r r r� � s r� c @ s| e Zd Zi Zej� Zddd�Zdd� Zf i fdd �Z d d � Z d d � Z e dd� �Z dd� Zdd� Zdd� Zdd� Zdd� ZdS )r NTFc C s� t j�8 t jj|jd �}|d kr:tj� t� f}|t j|j< W d Q R X |d | _|d | _ || _ | j j | _ || _ || _t| d | _|| _|d k r�tj|�| _n"| j d k r�| j j| _n tj� j| _|r�| j� tj| t j� d S )Nr r )r �_mutexr� r� r" r ZForkAwareLocalr� �_tls�_idset�_tokenr# �_id�_managerr� rW r� �_owned_by_managerr rU r� re rV �_increfr� � _after_fork) r$ r� r_ r� rV r� rP � manager_ownedZ tls_idsetr r r r% � s* zBaseProxy.__init__c C sd t jd� tj� j}tj� jdkr4|dtj� j 7 }| j| jj | j d�}t |d d|f� || j _ d S )Nz making connection to managerZ MainThread�|)rV rK )r rr r re r r\ r� r� r� r" r� r8 r� r )r$ r r� r r r �_connect� s zBaseProxy._connectc C s� y | j j}W n6 tk rB tjdtj� j� | j� | j j}Y nX |j | j |||f� |j � \}}|dkrp|S |dkr�|\}}| j j |j d } | jj|_| || j| j | j|d�} | j|j| jd�}t|d d|jf� | S t||��d S ) Nz#thread %r does not own a connectionz#RETURNz#PROXYr )r� rV r� )rV rQ ���)r� r r� r rr r\ r� r r� r/ r� r0 r� r� r! r� r" r� r� r� r8 r# r1 ) r$ r3 r4 r5 r� r6 r7 r� r� r� r� r r r � _callmethod� s, zBaseProxy._callmethodc C s | j d�S )Nz #GETVALUE)r� )r$ r r r � _getvalue s zBaseProxy._getvaluec C s� | j rtjd| jj� d S | j| jj| jd�}t|d d| j f� tjd| jj� | j j | j � | j oj| j j }tj| tj| j| j|| j| j | jfdd�| _d S )Nz%owned_by_manager skipped INCREF of %r)rV rP z INCREF %r� )r4 r� )r� r rr r� r# r� r" r� r8 r� r� �addr� r� r� r �_decrefr� Z_close)r$ r� r' r r r r� s zBaseProxy._increfc C s� |j | j� |d ks |jtjkr�y2tjd| j� || j|d�}t|d d| jf� W q� t k r� } ztjd|� W Y d d }~X q�X ntjd| j� | r�t |d�r�tjdt j � j � |jj� |`d S )Nz DECREF %r)rV rQ z... decref failed %sz%DECREF %r -- manager already shutdownr z-thread %r has no more proxies so closing conn)�discardr# r� r� r� r rr r" r8 rz r� r\ r� r r r| )r� rV r' ZtlsZidsetr� r� r� r r r r� s zBaseProxy._decrefc C sH d | _ y | j� W n0 tk rB } ztjd| � W Y d d }~X nX d S )Nzincref failed: %s)r� r� rz r r{ )r$ r� r r r r� 9 s zBaseProxy._after_forkc C s^ i }t � d k r| j|d< t| dd�rB| j|d< tt| j| j|ffS tt| �| j| j|ffS d S )NrV �_isautoFr� ) r r� r r� � RebuildProxyr� r� r� r )r$ r5 r r r r� A s zBaseProxy.__reduce__c C s | j � S )N)r� )r$ �memor r r � __deepcopy__N s zBaseProxy.__deepcopy__c C s dt | �j| jjt| �f S )Nz <%s object, typeid %r at %#x>)r r* r� r! r# )r$ r r r r+ Q s zBaseProxy.__repr__c C s4 y | j d�S tk r. t| �d d� d S X d S )Nr+ r z; '__str__()' failed>r� )r� rz r� )r$ r r r r? U s zBaseProxy.__str__)NNNTF)r* r, r- r� r ZForkAwareThreadLockr� r% r� r� r� r� r� r� r� r� r� r+ r? r r r r r � s ( c C s� t tj� dd �}|rT|j|jkrTtjd|� d|d< |j|jkrT|j|j |j|j< |j dd�opt tj� dd� }| ||fd|i|��S )Nrf z*Rebuild a proxy owned by manager, token=%rTr� rP Z _inheritingF) r r re r" r rr r# r[ rY �pop)rD r� r_ r5 r� rP r r r r� b s r� c C sr t |�}y || |f S tk r( Y nX i }x |D ]}td||f |� q4W t| tf|�}||_||| |f< |S )NzLdef %s(self, *args, **kwds): return self._callmethod(%r, args, kwds))r� r� �execr r r� )r r� �_cacheZdic�meth� ProxyTyper r r � MakeProxyTypew s r� Tc C s� t | d }|d krB|| j|d�}zt|d d| f�}W d |j� X |d krX|d k rX|j}|d krjtj� j}td| j |�}|| ||||d�} d| _ | S )Nr )rV rL z AutoProxy[%s])r� rV rP T) rW r" r8 r| r� r re rV r� r! r� ) r� r_ r� rV r� rP r� r� r� r� r r r r� � s r� c @ s e Zd Zdd� Zdd� ZdS )� Namespacec K s | j j|� d S )N)r� �update)r$ r5 r r r r% � s zNamespace.__init__c C s^ t | jj� �}g }x,|D ]$\}}|jd�s|jd||f � qW |j� d| jjdj|�f S )NrF z%s=%rz%s(%s)z, ) r r� r � startswithrB r� r) r* r� )r$ r rC r r� r r r r+ � s zNamespace.__repr__N)r* r, r- r% r+ r r r r r� � s r� c @ s8 e Zd Zd dd�Zdd� Zdd� Zdd � Zeee�Zd S ) �ValueTc C s || _ || _d S )N)� _typecode�_value)r$ r r� �lockr r r r% � s zValue.__init__c C s | j S )N)r )r$ r r r r� � s z Value.getc C s || _ d S )N)r )r$ r� r r r r� � s z Value.setc C s dt | �j| j| jf S )Nz %s(%r, %r))r r* r r )r$ r r r r+ � s zValue.__repr__N)T) r* r, r- r% r� r� r+ r� r� r r r r r� � s r� c C s t j | |�S )N)r )r Zsequencer r r r �Array� s r c @ s8 e Zd ZdZdd� Zdd� Zd d � Zd d � Zd d� ZdS )� IteratorProxy�__next__r/ �throwr| c C s | S )Nr )r$ r r r �__iter__� s zIteratorProxy.__iter__c G s | j d|�S )Nr )r� )r$ r4 r r r r � s zIteratorProxy.__next__c G s | j d|�S )Nr/ )r� )r$ r4 r r r r/ � s zIteratorProxy.sendc G s | j d|�S )Nr )r� )r$ r4 r r r r � s zIteratorProxy.throwc G s | j d|�S )Nr| )r� )r$ r4 r r r r| � s zIteratorProxy.closeN)r r/ r r| ) r* r, r- r� r r r/ r r| r r r r r � s r c @ s2 e Zd Zd Zddd�Zdd� Zd d � Zd d � ZdS )� AcquirerProxy�acquire�releaseTNc C s" |d kr|fn||f}| j d|�S )Nr )r� )r$ Zblockingr� r4 r r r r � s zAcquirerProxy.acquirec C s | j d�S )Nr )r� )r$ r r r r � s zAcquirerProxy.releasec C s | j d�S )Nr )r� )r$ r r r r� � s zAcquirerProxy.__enter__c C s | j d�S )Nr )r� )r$ r� r� r� r r r r� � s zAcquirerProxy.__exit__)r r )TN)r* r, r- r� r r r� r� r r r r r � s r c @ s4 e Zd ZdZddd�Zd d � Zd d � Zdd d�ZdS )�ConditionProxyr r rl �notify� notify_allNc C s | j d|f�S )Nrl )r� )r$ r� r r r rl � s zConditionProxy.waitc C s | j d�S )Nr )r� )r$ r r r r � s zConditionProxy.notifyc C s | j d�S )Nr )r� )r$ r r r r � s zConditionProxy.notify_allc C sh |� }|r|S |d k r$t j� | }nd }d }x6|sb|d k rP|t j� }|dkrPP | j|� |� }q.W |S )Nr )�timeZ monotonicrl )r$ Z predicater� r7 ZendtimeZwaittimer r r �wait_for� s zConditionProxy.wait_for)r r rl r r )N)N)r* r, r- r� rl r r r r r r r r � s r c @ s2 e Zd ZdZdd� Zdd� Zd d � Zdd d �Zd S )� EventProxyrk r� r� rl c C s | j d�S )Nrk )r� )r$ r r r rk s zEventProxy.is_setc C s | j d�S )Nr� )r� )r$ r r r r� s zEventProxy.setc C s | j d�S )Nr� )r� )r$ r r r r� s zEventProxy.clearNc C s | j d|f�S )Nrl )r� )r$ r� r r r rl s zEventProxy.wait)rk r� r� rl )N)r* r, r- r� rk r� r� rl r r r r r � s r c @ sN e Zd ZdZddd�Zdd � Zd d � Zed d � �Zedd� �Z edd� �Z dS )� BarrierProxy�__getattribute__rl �abort�resetNc C s | j d|f�S )Nrl )r� )r$ r� r r r rl s zBarrierProxy.waitc C s | j d�S )Nr )r� )r$ r r r r s zBarrierProxy.abortc C s | j d�S )Nr )r� )r$ r r r r s zBarrierProxy.resetc C s | j dd�S )Nr �parties)r )r� )r$ r r r r s zBarrierProxy.partiesc C s | j dd�S )Nr � n_waiting)r )r� )r$ r r r r s zBarrierProxy.n_waitingc C s | j dd�S )Nr �broken)r )r� )r$ r r r r s zBarrierProxy.broken)r rl r r )N) r* r, r- r� rl r r r� r r r r r r r r s r c @ s( e Zd Zd Zdd� Zdd� Zdd � Zd S ) �NamespaceProxyr � __setattr__� __delattr__c C s0 |d dkrt j| |�S t j| d�}|d|f�S )Nr rF r� r )�objectr )r$ r� � callmethodr r r � __getattr__ s zNamespaceProxy.__getattr__c C s4 |d dkrt j| ||�S t j| d�}|d||f�S )Nr rF r� r )r r r )r$ r� r� r r r r r % s zNamespaceProxy.__setattr__c C s0 |d dkrt j| |�S t j| d�}|d|f�S )Nr rF r� r )r r r )r$ r� r r r r r * s zNamespaceProxy.__delattr__N)r r r )r* r, r- r� r r r r r r r r s r c @ s* e Zd ZdZdd� Zdd� Zeee�ZdS ) � ValueProxyr� r� c C s | j d�S )Nr� )r� )r$ r r r r� 3 s zValueProxy.getc C s | j d|f�S )Nr� )r� )r$ r� r r r r� 5 s zValueProxy.setN)r� r� )r* r, r- r� r� r� r� r� r r r r r 1 s r � BaseListProxy�__add__� __contains__� __delitem__� __getitem__�__len__�__mul__� __reversed__�__rmul__� __setitem__rB �count�extend�index�insertr� �remove�reverser� �__imul__c @ s e Zd Zdd� Zdd� ZdS )� ListProxyc C s | j d|f� | S )Nr* )r� )r$ r� r r r �__iadd__A s zListProxy.__iadd__c C s | j d|f� | S )Nr/ )r� )r$ r� r r r r/ D s zListProxy.__imul__N)r* r, r- r1 r/ r r r r r0 @ s r0 � DictProxyr r� r� r� �has_key�popitem� setdefaultr� �Iterator� ArrayProxy� PoolProxy�apply� apply_asyncr| �imap�imap_unorderedr� �map� map_async�starmap� starmap_asyncr� Z AsyncResult)r: r> r@ r; r< c @ s e Zd Zdd� Zdd� ZdS )r8 c C s | S )Nr )r$ r r r r� d s zPoolProxy.__enter__c C s | j � d S )N)r� )r$ r� r� r� r r r r� f s zPoolProxy.__exit__N)r* r, r- r� r� r r r r r8 c s c @ s e Zd ZdS )r N)r* r, r- r r r r r m s �QueueZ JoinableQueuerc r] �RLock� Semaphore�BoundedSemaphore� Condition�Barrier�Poolr �dictF)r� r� )r� )r r r )NNNT)T)r r! r" r# r$ r% r& r' r( rB r) r* r+ r, r� r- r. r� r/ )r! r" r# r r$ r( r� r� r� r3 r r r� r4 r5 r� r )r$ r# r( ) r9 r: r| r; r< r� r= r> r? r@ r� )I�__all__ro r\ r Zqueuer r� r � r �contextr r r r r r r r� Z view_typesr r Z view_typer r r8 r1 rz r9 rE rG rH r� r` ra Z XmlListenerZ XmlClientrW r r� r� r r� r� r� r� r� r r r r r r r r r r0 r2 r� r7 Z BasePoolProxyr8 r rA rc r] rB rC rD rE rF rG rH r r r r �<module> s� ; ] 4 __pycache__/connection.cpython-36.opt-2.pyc 0000644 00000053450 15204243004 0014502 0 ustar 00 3 �I�iVy � @ s d dddgZ ddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl m Z ddl m Z m Z dd lmZ ejZy$ddlZdd lmZmZmZmZ W n$ ek r� ejd kr dZY nX d Zd ZdZej � Z dZ dgZ e ed��rdZ e dg7 Z ejd k�r dZ e dg7 Z efdd�Z!dd� Z"dd� Z#dd� Z$dd� Z%G d d � d �Z&e�rnG d d � d e&�Z'G d d!� d!e&�Z(G d"d� de)�Z*dPd#d �Z+ejd k�r�dQd%d�Z,n dRd&d�Z,G d'd(� d(e)�Z-d)d*� Z.ejd k�r�G d+d,� d,e)�Z/d-d.� Z0d/Z1d0Z2d1Z3d2Z4d3d4� Z5d5d6� Z6G d7d8� d8e)�Z7d9d:� Z8d;d<� Z9G d=d>� d>e*�Z:d?d@� Z;ejd k�rzdAdB� Z<ej=ej>hZ?dSdCd�Z@n,ddlAZAe eAdD��r�eAjBZCneAjDZCdTdEd�Z@ejd k�r�dFdG� ZEdHdI� ZFejGe(eE� dJdK� ZHdLdM� ZIejGe'eH� n dNdG� ZEdOdI� ZFejGe(eE� dS )U�Client�Listener�Pipe�wait� N� )�util)�AuthenticationError�BufferTooShort)� reduction)� WAIT_OBJECT_0�WAIT_ABANDONED_0� WAIT_TIMEOUT�INFINITE�win32i g 4@Zsha256�AF_INET�AF_UNIX�AF_PIPEc C s t j� | S )N)�time� monotonic)�timeout� r �2/usr/lib64/python3.6/multiprocessing/connection.py� _init_timeout? s r c C s t j� | kS )N)r r )�tr r r �_check_timeoutB s r c C sX | dkr d S | dkr&t jdtj� d�S | dkrLt jdtj� tt�f d d�S td ��d S ) Nr � localhostr r z listener-)�prefix�dirr z\\.\pipe\pyc-%d-%d-� zunrecognized family)r r ) �tempfileZmktempr Z get_temp_dir�os�getpid�next� _mmap_counter� ValueError)�familyr r r �arbitrary_addressI s r& c C sJ t jdkr | dkr td| ��t jdkrF| dkrFtt| �sFtd| ��d S )Nr r z Family %s is not recognized.r )�sys�platformr$ �hasattr�socket)r% r r r �_validate_familyW s r+ c C sJ t | �tkrdS t | �tkr*| jd�r*dS t | �tkr:dS td| ��d S )Nr z\\r r z address type of %r unrecognized)�type�tuple�str� startswithr$ )�addressr r r � address_typec s r1 c @ s� e Zd ZdZd+dd�Zdd� Zdd� Zd d � Zd d � Zd d� Z e dd� �Z e dd� �Z e dd� �Z dd� Zdd� Zd,dd�Zd d � Zd-d d �Zd.d d!�Zd"d#� Zd/d%d&�Zd'd(� Zd)d*� ZdS )0�_ConnectionBaseNTc C sB |j � }|dk rtd��| r,| r,td��|| _|| _|| _d S )Nr zinvalid handlez6at least one of `readable` and `writable` must be True)� __index__r$ �_handle� _readable� _writable)�self�handle�readable�writabler r r �__init__y s z_ConnectionBase.__init__c C s | j d k r| j� d S )N)r4 �_close)r7 r r r �__del__� s z_ConnectionBase.__del__c C s | j d krtd��d S )Nzhandle is closed)r4 �OSError)r7 r r r � _check_closed� s z _ConnectionBase._check_closedc C s | j std��d S )Nzconnection is write-only)r5 r> )r7 r r r �_check_readable� s z _ConnectionBase._check_readablec C s | j std��d S )Nzconnection is read-only)r6 r> )r7 r r r �_check_writable� s z _ConnectionBase._check_writablec C s" | j rd| _n| j� td��d S )NFzbad message length)r6 r5 �closer> )r7 r r r �_bad_message_length� s z#_ConnectionBase._bad_message_lengthc C s | j d kS )N)r4 )r7 r r r �closed� s z_ConnectionBase.closedc C s | j S )N)r5 )r7 r r r r9 � s z_ConnectionBase.readablec C s | j S )N)r6 )r7 r r r r: � s z_ConnectionBase.writablec C s | j � | jS )N)r? r4 )r7 r r r �fileno� s z_ConnectionBase.filenoc C s$ | j d k r z | j� W d d | _ X d S )N)r4 r< )r7 r r r rB � s z_ConnectionBase.closer c C s� | j � | j� t|�}|jdkr.tt|��}t|�}|dk rFtd��||k rVtd��|d krh|| }n&|dk rztd��n|| |kr�td��| j|||| � � d S )Nr r zoffset is negativezbuffer length < offsetzsize is negativez buffer length < offset + size)r? rA � memoryview�itemsize�bytes�lenr$ � _send_bytes)r7 �buf�offset�size�m�nr r r � send_bytes� s" z_ConnectionBase.send_bytesc C s$ | j � | j� | jtj|�� d S )N)r? rA rJ �_ForkingPickler�dumps)r7 �objr r r �send� s z_ConnectionBase.sendc C sJ | j � | j� |d k r(|dk r(td��| j|�}|d krB| j� |j� S )Nr znegative maxlength)r? r@ r$ � _recv_bytesrC �getvalue)r7 Z maxlengthrK r r r � recv_bytes� s z_ConnectionBase.recv_bytesc C s� | j � | j� t|���}|j}|t|� }|dk r>td��n||krNtd��| j� }|j� }||| k rvt|j � ��|j d� |j ||| || | � � |S Q R X d S )Nr znegative offsetzoffset too large) r? r@ rF rG rI r$ rU �tellr rV �seek�readinto)r7 rK rL rN rG Zbytesize�resultrM r r r �recv_bytes_into� s" z _ConnectionBase.recv_bytes_intoc C s& | j � | j� | j� }tj|j� �S )N)r? r@ rU rQ �loads� getbuffer)r7 rK r r r �recv� s z_ConnectionBase.recv� c C s | j � | j� | j|�S )N)r? r@ �_poll)r7 r r r r �poll s z_ConnectionBase.pollc C s | S )Nr )r7 r r r � __enter__ s z_ConnectionBase.__enter__c C s | j � d S )N)rB )r7 �exc_type� exc_value�exc_tbr r r �__exit__ s z_ConnectionBase.__exit__)TT)r N)N)r )r` )�__name__� __module__� __qualname__r4 r; r= r? r@ rA rC �propertyrD r9 r: rE rB rP rT rW r\ r_ rb rc rg r r r r r2 v s( r2 c @ s@ e Zd ZdZejfdd�Zdd� Zd dd�Zd d � Z d d � Z dS )�PipeConnectionFc C s || j � d S )N)r4 )r7 Z _CloseHandler r r r< s zPipeConnection._closec C sh t j| j|dd�\}}z<y |t jkr6t j|jgdt�}W n |j� � Y nX W d |jd�\}}X d S )NT)� overlappedF) �_winapiZ WriteFiler4 �ERROR_IO_PENDING�WaitForMultipleObjects�eventr �cancel�GetOverlappedResult)r7 rK �ov�err�waitresZnwrittenr r r rJ s zPipeConnection._send_bytesNc C s | j rd| _ tj� S |d kr dnt|d�}y�tj| j|dd�\}}z<y |tjkrbtj|j gdt �}W n |j � � Y nX W d |j d�\}}|dkr�tj� }|j |j� � |S |tjkr�| j||�S X W n8 tk �r } z|jtjkr�t�n� W Y d d }~X nX td��d S )NF� T)rm r z.shouldn't get here; expected KeyboardInterrupt)�_got_empty_message�io�BytesIO�minrn �ReadFiler4 ro rp rq r rr rs �writer^ ZERROR_MORE_DATA�_get_more_datar> �winerror�ERROR_BROKEN_PIPE�EOFError� RuntimeError) r7 �maxsizeZbsizert ru rv Znread�f�er r r rU * s6 zPipeConnection._recv_bytesc C s. | j stj| j�d dkr dS tt| g|��S )Nr T)rx rn � PeekNamedPiper4 �boolr )r7 r r r r ra J s zPipeConnection._pollc C s� |j � }tj� }|j|� tj| j�d }|d k rJt|�| |krJ| j� tj | j|dd�\}}|j d�\}}|j|j � � |S )Nr T)rm ) r^ ry rz r} rn r� r4 rI rC r| rs )r7 rt r� rK r� �leftru Zrbytesr r r r~ P s z PipeConnection._get_more_data)N) rh ri rj rx rn � CloseHandler< rJ rU ra r~ r r r r rl s rl c @ sx e Zd Zer(ejfdd�ZejZej Z ne j fdd�Ze j Ze jZ efdd�Ze fdd�Zdd � Zdd d �Zd d� Zd S )� Connectionc C s || j � d S )N)r4 )r7 r< r r r r< g s zConnection._closec C s || j � d S )N)r4 )r7 r< r r r r< l s c C s< t |�}x.|| j|�}||8 }|dkr(P ||d � }q W d S )Nr )rI r4 )r7 rK r} � remainingrO r r r �_sendq s zConnection._sendc C sf t j� }| j}|}xN|dkr`|||�}t|�}|dkrL||krDt�ntd��|j|� ||8 }qW |S )Nr z got end of file during message)ry rz r4 rI r� r> r} )r7 rM �readrK r8 r� �chunkrO r r r �_recvz s zConnection._recvc C sD t |�}tjd|�}|dkr2| j|� | j|� n| j|| � d S )Nz!ii @ )rI �structZpackr� )r7 rK rO �headerr r r rJ � s zConnection._send_bytesNc C s: | j d�}tjd|j� �\}|d k r0||kr0d S | j |�S )N� z!i)r� r� ZunpackrV )r7 r� rK rM r r r rU � s zConnection._recv_bytesc C s t | g|�}t|�S )N)r r� )r7 r �rr r r ra � s zConnection._poll)N)rh ri rj rn �_multiprocessingZ closesocketr< rT Z_writer_ Z_readr rB r} r� r� r� rJ rU ra r r r r r� ` s r� c @ sN e Zd Zddd�Zdd� Zdd� Zed d � �Zed d � �Zd d � Z dd� Z dS )r Nr c C sr |p|rt |�pt}|p t|�}t|� |dkr>t||�| _nt|||�| _|d k rht|t� rht d��|| _ d S )Nr z authkey should be a byte string) r1 �default_familyr& r+ � PipeListener� _listener�SocketListener� isinstancerH � TypeError�_authkey)r7 r0 r% �backlog�authkeyr r r r; � s zListener.__init__c C s> | j d krtd��| j j� }| jr:t|| j� t|| j� |S )Nzlistener is closed)r� r> �acceptr� �deliver_challenge�answer_challenge)r7 �cr r r r� � s zListener.acceptc C s | j }|d k r d | _ |j� d S )N)r� rB )r7 Zlistenerr r r rB � s zListener.closec C s | j jS )N)r� �_address)r7 r r r �<lambda>� s zListener.<lambda>c C s | j jS )N)r� �_last_accepted)r7 r r r r� � s c C s | S )Nr )r7 r r r rc � s zListener.__enter__c C s | j � d S )N)rB )r7 rd re rf r r r rg � s zListener.__exit__)NNr N) rh ri rj r; r� rB rk r0 Z last_acceptedrc rg r r r r r � s c C sj |p t | �}t|� |dkr&t| �}nt| �}|d k rJt|t� rJtd��|d k rft||� t||� |S )Nr z authkey should be a byte string) r1 r+ � PipeClient� SocketClientr� rH r� r� r� )r0 r% r� r� r r r r � s Tc C sj | r>t j� \}}|jd� |jd� t|j� �}t|j� �}n$tj� \}}t|dd�}t|dd�}||fS )NTF)r: )r9 )r* Z socketpair� setblockingr� �detachr �pipe)�duplex�s1�s2�c1�c2Zfd1Zfd2r r r r � s c C s� t d�}| r*tj}tjtjB }tt }}ntj}tj}dt }}tj||tjB tj B tj tj B tj B d||tj tj�}tj||dtjtjtjtj�}tj|tj d d � tj|dd�}|jd�\} } t|| d�} t|| d�} | | fS )Nr r r T)rm )r: )r9 )r& rn �PIPE_ACCESS_DUPLEX� GENERIC_READ� GENERIC_WRITE�BUFSIZEZPIPE_ACCESS_INBOUND�CreateNamedPipe�FILE_FLAG_OVERLAPPED� FILE_FLAG_FIRST_PIPE_INSTANCE�PIPE_TYPE_MESSAGE�PIPE_READMODE_MESSAGE� PIPE_WAIT�NMPWAIT_WAIT_FOREVER�NULL� CreateFile� OPEN_EXISTING�SetNamedPipeHandleState�ConnectNamedPipers rl ) r� r0 Zopenmode�accessZobsizeZibsizeZh1Zh2rm �_ru r� r� r r r r s2 c @ s& e Zd Zd dd�Zdd� Zdd� ZdS ) r� r c C s� t j tt |��| _yRtjdkr2| jjt jt jd� | jjd� | jj |� | jj |� | jj � | _ W n t k r� | jj� � Y nX || _d | _|dkr�tj| tj|fdd�| _nd | _d S )N�posixr Tr r )�args� exitpriority)r* �getattr�_socketr �nameZ setsockoptZ SOL_SOCKETZ SO_REUSEADDRr� ZbindZlistenZ getsocknamer� r> rB Z_familyr� r �Finalize�unlink�_unlink)r7 r0 r% r� r r r r; < s$ zSocketListener.__init__c C s& | j j� \}| _|jd� t|j� �S )NT)r� r� r� r� r� r� )r7 �sr r r r� T s zSocketListener.acceptc C s0 z| j j� W d | j}|d k r*d | _|� X d S )N)r� rB r� )r7 r� r r r rB Y s zSocketListener.closeN)r )rh ri rj r; r� rB r r r r r� 8 s r� c C sD t | �}tjtt|���"}|jd� |j| � t|j� �S Q R X d S )NT)r1 r* r� r� Zconnectr� r� )r0 r% r� r r r r� c s r� c @ s4 e Zd Zd dd�Zd dd�Zdd� Zed d � �ZdS ) r� Nc C sL || _ | jdd�g| _d | _tjd| j � tj| tj| j| j fdd�| _ d S )NT)�firstz listener created with address=%rr )r� r� ) r� � _new_handle� _handle_queuer� r � sub_debugr� r� �_finalize_pipe_listenerrB )r7 r0 r� r r r r; w s zPipeListener.__init__Fc C sH t jt jB }|r|t jO }t j| j|t jt jB t jB t j t t t j t j �S )N) rn r� r� r� r� r� r� r� r� ZPIPE_UNLIMITED_INSTANCESr� r� r� )r7 r� �flagsr r r r� � s zPipeListener._new_handlec C s� | j j| j� � | j jd�}ytj|dd�}W n0 tk r^ } z|jtjkrN� W Y d d }~X nPX z<ytj |j gdt �}W n |j � tj |� � Y nX W d |jd�\}}X t|�S )Nr T)rm F)r� �appendr� �poprn r� r> r Z ERROR_NO_DATArp rq r rr r� rs rl )r7 r8 rt r� �resr� ru r r r r� � s" zPipeListener.acceptc C s( t jd|� x| D ]}tj|� qW d S )Nz closing listener with address=%r)r r� rn r� )Zqueuer0 r8 r r r r� � s z$PipeListener._finalize_pipe_listener)N)F)rh ri rj r; r� r� � staticmethodr� r r r r r� s s r� c C s� t � }x~y6tj| d� tj| tjtjB dtjtjtjtj�}W n> t k r| } z"|j tj tj fksjt |�rl� W Y d d }~X qX P qW � tj|tjd d � t|�S )Ni� r )r rn Z WaitNamedPiper� r� r� r� r� r� r> r ZERROR_SEM_TIMEOUTZERROR_PIPE_BUSYr r� r� rl )r0 r �hr� r r r r� � s" r� � s #CHALLENGE#s #WELCOME#s #FAILURE#c C sf dd l }tjt�}| jt| � |j||t�j� }| j d�}||krP| jt � n| jt � t d��d S )Nr � zdigest received was wrong) �hmacr �urandom�MESSAGE_LENGTHrP � CHALLENGE�new�HMAC_DIGEST_NAME�digestrW �WELCOME�FAILUREr )� connectionr� r� �messager� �responser r r r� � s r� c C s\ dd l }| jd�}|tt�d � }|j||t�j� }| j|� | jd�}|tkrXt d��d S )Nr r� zdigest sent was rejected) r� rW rI r� r� r� r� rP r� r )r� r� r� r� r� r� r r r r� � s r� c @ s$ e Zd Zdd� Zdd� Zdd� ZdS )�ConnectionWrapperc C s: || _ || _|| _x"dD ]}t||�}t| ||� qW d S )NrE rB rb rW rP )rE rB rb rW rP )�_conn�_dumps�_loadsr� �setattr)r7 �connrR r] �attrrS r r r r; � s zConnectionWrapper.__init__c C s | j |�}| jj|� d S )N)r� r� rP )r7 rS r� r r r rT � s zConnectionWrapper.sendc C s | j j� }| j|�S )N)r� rW r� )r7 r� r r r r_ � s zConnectionWrapper.recvN)rh ri rj r; rT r_ r r r r r� � s r� c C s t j| fd d d d�jd�S )Nr zutf-8)� xmlrpclibrR �encode)rS r r r � _xml_dumps� s r� c C s t j| jd��\\}}|S )Nzutf-8)r� r] �decode)r� rS �methodr r r � _xml_loads� s r� c @ s e Zd Zdd� ZdS )� XmlListenerc C s dd l jatj| �}t|tt�S )Nr )� xmlrpc.client�clientr� r r� r� r� r� )r7 rS r r r r� s zXmlListener.acceptN)rh ri rj r� r r r r r� s r� c O s dd l jatt| |�tt�S )Nr )r� r� r� r� r r� r� )r� �kwdsr r r � XmlClient s r� c C s� t | �}g }x�|r�tj|d|�}|tkr,P n\t| koFtt|� k n rV|t8 }n2t| koptt|� k n r�|t8 }ntd��|j|| � ||d d � }d}qW |S )NFzShould not get herer r ) �listrn rp r r rI r r� r� )Zhandlesr �L�readyr� r r r �_exhaustive_wait s r� c . sn |d krt }n |dk r d}nt|d d �}t| �} i �g }t� � t� }�zH�x2| D �](}yt|d�}W n tk r� |�|j� < Y qTX ytj|� dd�\}}W n8 t k r� } z d |j }}|t krʂ W Y d d }~X nX |tj kr�|j |� |�|j< qT|�rptj� d d� dk�rpy|jd�\} }W n* t k �rT } z |j }W Y d d }~X nX | �rpt|d ��rpd|_� j|� d}qTW t�j� |�}W d x|D ]}|j� �q�W x�|D ]�}y|jd�\} }W n6 t k �r� } z|j }|t k�r� W Y d d }~X nX |tjk�r��|j }� j|� |dk�r�t|d ��r�d|_�q�W X � j�fd d �|D �� � fd d �| D �S )Nr i� g �?rE T� � Frx c 3 s | ]}� | V qd S )Nr )�.0r� )�waithandle_to_objr r � <genexpr>v s zwait.<locals>.<genexpr>c s g | ]}|� kr|�qS r r )r� �o)� ready_objectsr r � <listcomp>w s zwait.<locals>.<listcomp>)r� r� )r �intr� �setr� �AttributeErrorr3 rn r| r> r � _ready_errorsro r� rq r' Zgetwindowsversionrs r) rx �addr� �keysrr ZERROR_OPERATION_ABORTED�update) � object_listr Zov_listZ ready_handlesr� rE rt ru r� r� r )r r� r r + sh � PollSelectorc C s� t � �x}x| D ]}|j|tj� qW |d k r8tj� | }x@|j|�}|rVdd� |D �S |d k r:|tj� }|dk r:|S q:W W d Q R X d S )Nc S s g | ]\}}|j �qS r )Zfileobj)r� �keyZeventsr r r r � s zwait.<locals>.<listcomp>r )� _WaitSelector�register� selectorsZ EVENT_READr r Zselect)r r ZselectorrS Zdeadliner� r r r r � s c C sN | j � }tj|tjtj��*}ddlm} |j|�}t|| j | j ffS Q R X d S )Nr )�resource_sharer) rE r* Zfromfdr Z SOCK_STREAMr r Z DupSocket�rebuild_connectionr9 r: )r� r8 r� r �dsr r r �reduce_connection� s r c C s | j � }t|j � ||�S )N)r� r� )r r9 r: Zsockr r r r � s r c C sB | j r tjnd| jrtjndB }tj| j� |�}t|| j | jffS )Nr ) r9 rn ZFILE_GENERIC_READr: ZFILE_GENERIC_WRITEr Z DupHandlerE �rebuild_pipe_connection)r� r� �dhr r r �reduce_pipe_connection� s r c C s | j � }t|||�S )N)r� rl )r r9 r: r8 r r r r � s r c C s t j| j� �}t|| j| jffS )N)r ZDupFdrE r r9 r: )r� �dfr r r r � s c C s | j � }t|||�S )N)r� r� )r r9 r: �fdr r r r � s )NN)T)T)N)N)J�__all__ry r r' r* r� r r � itertoolsr� r r r r �contextr ZForkingPicklerrQ rn r r r r � ImportErrorr( r� ZCONNECTION_TIMEOUTr� �countr# r� Zfamiliesr) r r r& r+ r1 r2 rl r� �objectr r r r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� ZERROR_NETNAME_DELETEDr r r r r ZSelectSelectorr r r r r r r r r �<module> s� PJ8 ,+ 8 P __pycache__/managers.cpython-36.pyc 0000644 00000101052 15204243005 0013171 0 ustar 00 3 \� � @ s� d dddgZ ddlZddlZddlZddlZddlZddlmZ ddlm Z dd l m Z m Z dd lm Z dd lmZ dd lmZ dd lmZ dd� Ze jeje� dd� d�D �Zed ek r�dd� ZxeD ]Ze jee� q�W G dd� de�Zf i fdd�Zdd� ZG d d � d e�Z d d � Z d d!� Z G d"d#� d#e�Z G d$d%� d%e�Z e j!e j"fe j#e j$fd&�Z%G d'd � d e�Z&G d(d)� d)e'�Z(G d*d� de�Z)d+d,� Z*i fd-d.�Z+d�d0d1�Z,G d2d3� d3e�Z-G d4d5� d5e�Z.d�d6d7�Z/G d8d9� d9e)�Z0G d:d;� d;e)�Z1G d<d=� d=e1�Z2G d>d?� d?e)�Z3G d@dA� dAe)�Z4G dBdC� dCe)�Z5G dDdE� dEe)�Z6e+dFd��Z7G dZd[� d[e7�Z8e+d\d��Z9d]deie9_:e+dfd��Z;e+dgd��Z<dsdsdsdededt�e<_:G dudg� dge<�Z=G dvd� de&�Z>e>jdwej?� e>jdxej?� e>jdyej@e3� e>jdzejAe1� e>jd{ejBe1� e>jd|ejCe1� e>jd}ejDe1� e>jd~ejEe2� e>jdejFe4� e>jd�e jGe=� e>jd�ee8� e>jd�eHe9� e>jd5e.e6� e>jd7e/e;� e>jd3e-e5� e>jdee0d�d�� e>jdsd�d�� dS )�� BaseManager� SyncManager� BaseProxy�Token� N)� format_exc� )� connection)� reduction�get_spawning_popen)�pool)�process)�util)� get_contextc C s t j | j| j� ffS )N)�array�typecode�tobytes)�a� r �0/usr/lib64/python3.6/multiprocessing/managers.py� reduce_array$ s r c C s g | ]}t ti |�� ��qS r )�type�getattr)�.0�namer r r � <listcomp>( s r �items�keys�valuesc C s t t | �ffS )N)�list)�objr r r �rebuild_as_list* s r c @ s4 e Zd ZdZdZdd� Zdd� Zd d � Zd d � Zd S )r z4 Type to uniquely indentify a shared object �typeid�address�idc C s ||| | _ | _| _d S )N)r! r" r# )�selfr! r" r# r r r �__init__9 s zToken.__init__c C s | j | j| jfS )N)r! r" r# )r$ r r r � __getstate__< s zToken.__getstate__c C s |\| _ | _| _d S )N)r! r" r# )r$ �stater r r � __setstate__? s zToken.__setstate__c C s d| j j| j| j| jf S )Nz %s(typeid=%r, address=%r, id=%r))� __class__�__name__r! r" r# )r$ r r r �__repr__B s zToken.__repr__N)r! r" r# ) r* � __module__� __qualname__�__doc__� __slots__r% r& r( r+ r r r r r 3 s c C s8 | j ||||f� | j� \}}|dkr*|S t||��dS )zL Send a message to manager using connection `c` and return response z#RETURNN)�send�recv�convert_to_error)�cr# � methodname�args�kwds�kind�resultr r r �dispatchJ s r9 c C s\ | dkr |S | dkr,t |�tks$t�t|�S | dkrPt |�tksDt�td| �S td�S d S )Nz#ERRORz #TRACEBACKz#UNSERIALIZABLEzUnserializable message: %s zUnrecognized message type)r �str�AssertionError� RemoteError� ValueError)r7 r8 r r r r2 T s r2 c @ s e Zd Zdd� ZdS )r< c C s& ddd d t | jd � dd S )N� �-�K r )r: r5 )r$ r r r �__str__a s zRemoteError.__str__N)r* r, r- rA r r r r r< ` s r< c C s6 g }x,t | �D ] }t| |�}t|�r|j|� qW |S )z4 Return a list of names of methods of `obj` )�dirr �callable�append)r �tempr �funcr r r � all_methodsh s rG c C s dd� t | �D �S )zP Return a list of names of methods of `obj` which do not start with '_' c S s g | ]}|d dkr|�qS )r �_r )r r r r r r w s z"public_methods.<locals>.<listcomp>)rG )r r r r �public_methodss s rI c @ s� e Zd ZdZdddddddd d g Zd d � Zd d� Zdd� Zdd� Zdd� Z dd� Z dd� Z dd� Z e e e d�Z d d � Zd d � Zd d!� Zd"d#� Zd$d%� Zd&d'� Zd(d)� Zd*d+� Zd,d-� Zd.S )/�ServerzM Server class which runs in a process controlled by a manager object �shutdown�create�accept_connection� get_methods� debug_info�number_of_objects�dummy�incref�decrefc C sl t |t�st�|| _tj|�| _t| \}}||dd�| _| jj | _ dd f fi| _ i | _ i | _ t j� | _d S )N� )r" Zbacklog�0)� isinstance�bytesr; �registryr �AuthenticationString�authkey�listener_client�listenerr" � id_to_obj�id_to_refcount�id_to_local_proxy_obj� threading�Lock�mutex)r$ rX r" rZ � serializer�Listener�Clientr r r r% � s zServer.__init__c C s� t j� | _| tj� _zZt j| jd�}d|_|j � y x| jj � sN| jj d� q6W W n t t fk rj Y nX W dtjtjkr�tjd� tjt_tjt_tjd� X dS )z( Run the server forever )�targetTr Nzresetting stdout, stderrr )r` �Event� stop_eventr �current_process�_manager_server�Thread�accepter�daemon�start�is_set�wait�KeyboardInterrupt� SystemExit�sys�stdout� __stdout__r �debug� __stderr__�stderr�exit)r$ rl r r r � serve_forever� s zServer.serve_foreverc C sP xJy| j j� }W n tk r& wY nX tj| j|fd�}d|_|j� qW d S )N)rf r5 T)r\ Zaccept�OSErrorr` rk �handle_requestrm rn )r$ r3 �tr r r rl � s zServer.accepterc ' C sL d } }}yTt j|| j� t j|| j� |j� }|\}}}}|| jksTtd| ��t| |�}W n tk r~ dt � f} Y n>X y||f|�|�}W n tk r� dt � f} Y n X d|f} y|j | � W nt tk �r> } zVy|j dt � f� W n tk �r Y nX t j d| � t j d|� t j d| � W Y dd} ~ X nX |j � dS )z) Handle a new connection Nz%r unrecognizedz #TRACEBACKz#RETURNzFailure to send message: %rz ... request was %rz ... exception was %r)r Zdeliver_challengerZ Zanswer_challenger1 �publicr; r � Exceptionr r0 r �info�close) r$ r3 �funcnamer8 �request�ignorer5 r6 rF �msg�er r r r| � s4 zServer.handle_requestc D C s� t jdtj� j� |j}|j}| j}�x�| jj � �s�yBd }}|� }|\}}} } y|| \}} } W n^ t k r� } zBy| j | \}} } W n& t k r� } z | �W Y dd}~X nX W Y dd} ~ X nX || kr�t d|t |�| f ��t||�}y|| | �}W n, tk �r& } zd|f}W Y dd}~X nPX | �o8| j|d�}|�rn| j|||�\}}t|| j|�}d||ff}nd|f}W n� t k �r� |dk�r�dt� f}nNy,| j| }|| |||f| �| �}d|f}W n tk �r� dt� f}Y nX Y nP tk �r" t jdtj� j� tjd � Y n tk �r@ dt� f}Y nX yDy ||� W n2 tk �r� } z|d t� f� W Y dd}~X nX W q( tk �r� } zBt jd tj� j� t jd |� t jd |� |j� tjd� W Y dd}~X q(X q(W dS )zQ Handle requests from the proxies in a particular process/thread z$starting server thread to service %rNz+method %r of %r object is not in exposed=%rz#ERRORz#PROXYz#RETURNz #TRACEBACKz$got EOF -- exiting thread serving %rr z#UNSERIALIZABLEz exception in thread serving %rz ... message was %rz ... exception was %rr )r rv r` �current_threadr r1 r0 r] rh ro �KeyErrorr_ �AttributeErrorr r r �getrL r r" r �fallback_mapping�EOFErrorrs ry r� r� )r$ �connr1 r0 r] r4 r r� �identr5 r6 �exposed� gettypeid�keZ second_keZfunction�resr� r� r! ZridentZrexposed�tokenZ fallback_funcr8 r r r � serve_client� sx ( $ zServer.serve_clientc C s |S )Nr )r$ r� r� r r r r �fallback_getvalue& s zServer.fallback_getvaluec C s t |�S )N)r: )r$ r� r� r r r r � fallback_str) s zServer.fallback_strc C s t |�S )N)�repr)r$ r� r� r r r r � fallback_repr, s zServer.fallback_repr)rA r+ z #GETVALUEc C s d S )Nr )r$ r3 r r r rQ 5 s z Server.dummyc C s| | j �l g }t| jj� �}|j� xD|D ]<}|dkr(|jd|| j| t| j| d �dd� f � q(W dj|�S Q R X dS )zO Return some info --- useful to spot problems with refcounting rU z %s: refcount=%s %sr Nr@ r> ) rb r r^ r �sortrD r: r] �join)r$ r3 r8 r r� r r r rO 8 s $zServer.debug_infoc C s t | j�S )z* Number of shared objects )�lenr^ )r$ r3 r r r rP G s zServer.number_of_objectsc C sL z:yt jd� |jd� W n ddl}|j� Y nX W d| jj� X dS )z' Shutdown this process z!manager received shutdown message�#RETURNNr )r� N)r rv r0 � traceback� print_excrh �set)r$ r3 r� r r r rK N s zServer.shutdownc O s� | j �� | j| \}}}}|dkrBt|�dkr4| s8t�|d } n |||�} |dkr\t| �}|dk r�t|�tkstt�t|�t|� }dt| � } t j d|| � | t |�|f| j | < | | j kr�d| j | < W dQ R X | j|| � | t|�fS )z> Create a new shared object and return its id Nr r z%xz&%r callable returned object with id %r)rb rX r� r; rI r �dictr r# r rv r� r] r^ rR �tuple) r$ r3 r! r5 r6 rC r� �method_to_typeid� proxytyper r� r r r rL [ s$ z Server.createc C s t | j|j d �S )zL Return the methods of the shared object indicated by token r )r� r] r# )r$ r3 r� r r r rN z s zServer.get_methodsc C s" |t j� _|jd� | j|� dS )z= Spawn a new thread to serve this connection �#RETURNN)r� N)r` r� r r0 r� )r$ r3 r r r r rM � s zServer.accept_connectionc C s� | j �� y| j| d7 < W nh tk r� } zL|| jkrrd| j|< | j| | j|< | j| \}}}tjd|� n|�W Y d d }~X nX W d Q R X d S )Nr z&Server re-enabled tracking & INCREF %r)rb r^ r� r_ r] r rv )r$ r3 r� r� r r� r� r r r rR � s z Server.increfc C s� || j kr$|| jkr$tjd|� d S | j�@ | j | dks>t�| j | d8 < | j | dkrf| j |= W d Q R X || j kr�d f d f| j|< tjd|� | j� | j|= W d Q R X d S )NzServer DECREF skipping %rr r zdisposing of obj with id %r)r^ r_ r rv rb r; r] )r$ r3 r� r r r rS � s z Server.decrefN)r* r, r- r. r~ r% rz rl r| r� r� r� r� r� rQ rO rP rK rL rN rM rR rS r r r r rJ } s. "Q rJ c @ s e Zd ZdgZdZdZdZdS )�State�valuer r � N)r* r, r- r/ �INITIAL�STARTED�SHUTDOWNr r r r r� � s r� )�pickleZ xmlrpclibc @ s� e Zd ZdZi ZeZd!dd�Zdd� Zdd � Z df fd d �Z e df fd d ��Z dd� Z d"dd�Zdd� Zdd� Zdd� Zdd� Zedd� �Zed d � �Ze d#d d ��ZdS )$r z! Base class for managers Nr� c C s\ |d krt j� j}|| _t j|�| _t� | _tj| j_ || _ t | \| _ | _ |pTt� | _d S )N)r ri rZ �_addressrY �_authkeyr� �_stater� r� � _serializerr[ Z _Listener�_Clientr �_ctx)r$ r" rZ rc Zctxr r r r% � s zBaseManager.__init__c C s( | j jtjkst�t| j| j| j| j �S )zX Return server object with serve_forever() method and address attribute ) r� r� r� r� r; rJ � _registryr� r� r� )r$ r r r � get_server� s zBaseManager.get_serverc C s8 t | j \}}|| j| jd�}t|dd� tj| j_dS )z> Connect manager object to the server process )rZ NrQ ) r[ r� r� r� r9 r� r� r� r� )r$ rd re r� r r r �connect� s zBaseManager.connectc C s� | j jtjkst�|dk r,t|� r,td��tjdd�\}}| j j t | �j | j | j| j| j|||fd�| _djdd� | jjD ��}t | �jd | | j_| jj� |j� |j� | _|j� tj| j _tj| t | �j | j| j| j| j | j fd d �| _ dS ) z@ Spawn a server process for this manager object Nz initializer must be a callableF)Zduplex)rf r5 �:c s s | ]}t |�V qd S )N)r: )r �ir r r � <genexpr>� s z$BaseManager.start.<locals>.<genexpr>r? r )r5 � exitpriority) r� r� r� r� r; rC � TypeErrorr ZPiper� ZProcessr � _run_serverr� r� r� r� �_processr� Z _identityr* r rn r� r1 r� r �Finalize�_finalize_managerr� rK )r$ � initializer�initargs�reader�writerr� r r r rn � s( zBaseManager.startc C sN |dk r||� | j ||||�}|j|j� |j� tjd|j� |j� dS )z@ Create a server, report its address and run it Nzmanager serving at %r)�_Serverr0 r" r� r r� rz ) �clsrX r" rZ rc r� r� r� �serverr r r r� s zBaseManager._run_serverc O sd | j jtjkstd��| j| j| jd�}z t|dd|f| |�\}}W d|j � X t || j|�|fS )zP Create a new shared object; return the token and exposed tuple zserver not yet started)rZ NrL ) r� r� r� r� r; r� r� r� r9 r� r )r$ r! r5 r6 r� r# r� r r r �_create% s zBaseManager._createc C s* | j dk r&| j j|� | j j� s&d| _ dS )zC Join the manager process (if it has been spawned) N)r� r� �is_alive)r$ �timeoutr r r r� 1 s zBaseManager.joinc C s. | j | j| jd�}z t|dd�S |j� X dS )zS Return some info about the servers shared objects and connections )rZ NrO )r� r� r� r9 r� )r$ r� r r r � _debug_info: s zBaseManager._debug_infoc C s. | j | j| jd�}z t|dd�S |j� X dS )z5 Return the number of shared objects )rZ NrP )r� r� r� r9 r� )r$ r� r r r �_number_of_objectsD s z BaseManager._number_of_objectsc C s, | j jtjkr| j� | j jtjks(t�| S )N)r� r� r� r� rn r� r; )r$ r r r � __enter__N s zBaseManager.__enter__c C s | j � d S )N)rK )r$ �exc_type�exc_val�exc_tbr r r �__exit__T s zBaseManager.__exit__c C s� | j � r�tjd� y,|||d�}zt|dd� W d|j� X W n tk rR Y nX | jdd� | j � r�tjd� t| d�r�tjd � | j� | jd d� | j � r�tjd � t j |_ y t j |= W n tk r� Y nX dS ) zQ Shutdown the manager process; will be registered as a finalizer z#sending shutdown message to manager)rZ NrK g �?)r� zmanager still alive� terminatez'trying to `terminate()` manager processg�������?z#manager still alive after terminate)r� r r� r9 r� r r� �hasattrr� r� r� r� r �_address_to_localr� )r r" rZ r' r� r� r r r r� W s. z BaseManager._finalize_managerc C s | j S )N)r� )r$ r r r �<lambda>w s zBaseManager.<lambda>Tc s� d| j kr| jj� | _� dkr"t� |p0t� dd�}|p@t� dd�}|r�xHt|j� �D ]8\}}t|�tkstt d| ��t|�tksTt d| ��qTW |||� f| j�< |rƇ �fdd�} �| _ t | �| � dS )z9 Register a typeid with the manager type r� N� _exposed_�_method_to_typeid_z%r is not a stringc s` t jd�� | j�f|�|�\}}� || j| | j|d�}| j|j| jd�}t|d d|jf� |S )Nz)requesting creation of a shared %r object)�managerrZ r� )rZ rS ) r rv r� r� r� r� r" r9 r# )r$ r5 r6 r� Zexp�proxyr� )r� r! r r rE � s z"BaseManager.register.<locals>.temp) �__dict__r� �copy� AutoProxyr r r r r: r; r* �setattr) r� r! rC r� r� r� � create_method�keyr� rE r )r� r! r �registery s zBaseManager.register)NNr� N)N)NNNNT)r* r, r- r. r� rJ r� r% r� r� rn � classmethodr� r� r� r� r� r� r� � staticmethodr� �propertyr" r� r r r r r � s* $ c @ s e Zd Zdd� Zdd� ZdS )�ProcessLocalSetc C s t j| dd� � d S )Nc S s | j � S )N)�clear)r r r r r� � s z*ProcessLocalSet.__init__.<locals>.<lambda>)r �register_after_fork)r$ r r r r% � s zProcessLocalSet.__init__c C s t | �f fS )N)r )r$ r r r � __reduce__� s zProcessLocalSet.__reduce__N)r* r, r- r% r� r r r r r� � s r� c @ s� e Zd ZdZi Zej� Zddd�Zdd� Z f i fd d �Z d d � Z d d� Z e dd� �Zdd� Zdd� Zdd� Zdd� Zdd� ZdS ) r z. A base for proxies of shared objects NTFc C s� t j�8 t jj|jd �}|d kr:tj� t� f}|t j|j< W d Q R X |d | _|d | _ || _ | j j | _ || _ || _t| d | _|| _|d k r�tj|�| _n"| j d k r�| j j| _n tj� j| _|r�| j� tj| t j� d S )Nr r )r �_mutexr� r� r" r ZForkAwareLocalr� �_tls�_idset�_tokenr# �_id�_managerr� r[ r� �_owned_by_managerr rY r� ri rZ �_increfr� � _after_fork) r$ r� rc r� rZ r� rR � manager_ownedZ tls_idsetr r r r% � s* zBaseProxy.__init__c C sd t jd� tj� j}tj� jdkr4|dtj� j 7 }| j| jj | j d�}t |d d|f� || j _ d S )Nz making connection to managerZ MainThread�|)rZ rM )r rv r ri r r` r� r� r� r" r� r9 r� r )r$ r r� r r r �_connect� s zBaseProxy._connectc C s� y | j j}W n6 tk rB tjdtj� j� | j� | j j}Y nX |j | j |||f� |j � \}}|dkrp|S |dkr�|\}}| j j |j d } | jj|_| || j| j | j|d�} | j|j| jd�}t|dd|jf� | S t||��dS ) zW Try to call a method of the referrent and return a copy of the result z#thread %r does not own a connectionz#RETURNz#PROXYr )r� rZ r� )rZ NrS ���)r� r r� r rv r` r� r r� r0 r� r1 r� r� r! r� r" r� r� r� r9 r# r2 ) r$ r4 r5 r6 r� r7 r8 r� r� r� r� r r r � _callmethod� s, zBaseProxy._callmethodc C s | j d�S )z9 Get a copy of the value of the referent z #GETVALUE)r� )r$ r r r � _getvalue s zBaseProxy._getvaluec C s� | j rtjd| jj� d S | j| jj| jd�}t|d d| j f� tjd| jj� | j j | j � | j oj| j j }tj| tj| j| j|| j| j | jfdd�| _d S )Nz%owned_by_manager skipped INCREF of %r)rZ rR z INCREF %r� )r5 r� )r� r rv r� r# r� r" r� r9 r� r� �addr� r� r� r �_decrefr� Z_close)r$ r� r' r r r r� s zBaseProxy._increfc C s� |j | j� |d ks |jtjkr�y2tjd| j� || j|d�}t|d d| jf� W q� t k r� } ztjd|� W Y d d }~X q�X ntjd| j� | r�t |d�r�tjdt j � j � |jj� |`d S )Nz DECREF %r)rZ rS z... decref failed %sz%DECREF %r -- manager already shutdownr z-thread %r has no more proxies so closing conn)�discardr# r� r� r� r rv r" r9 r r� r` r� r r r� )r� rZ r' ZtlsZidsetr� r� r� r r r r� s zBaseProxy._decrefc C sH d | _ y | j� W n0 tk rB } ztjd| � W Y d d }~X nX d S )Nzincref failed: %s)r� r� r r r� )r$ r� r r r r� 9 s zBaseProxy._after_forkc C s^ i }t � d k r| j|d< t| dd�rB| j|d< tt| j| j|ffS tt| �| j| j|ffS d S )NrZ �_isautoFr� ) r r� r r� � RebuildProxyr� r� r� r )r$ r6 r r r r� A s zBaseProxy.__reduce__c C s | j � S )N)r� )r$ �memor r r � __deepcopy__N s zBaseProxy.__deepcopy__c C s dt | �j| jjt| �f S )Nz <%s object, typeid %r at %#x>)r r* r� r! r# )r$ r r r r+ Q s zBaseProxy.__repr__c C s4 y | j d�S tk r. t| �dd� d S X dS )zV Return representation of the referent (or a fall-back if that fails) r+ Nr z; '__str__()' failed>r� )r� r r� )r$ r r r rA U s zBaseProxy.__str__)NNNTF)r* r, r- r. r� r ZForkAwareThreadLockr� r% r� r� r� r� r� r� r� r� r� r+ rA r r r r r � s ( c C s� t tj� dd�}|rT|j|jkrTtjd|� d|d< |j|jkrT|j|j |j|j< |j dd�opt tj� dd� }| ||fd|i|��S ) z5 Function used for unpickling proxy objects. rj Nz*Rebuild a proxy owned by manager, token=%rTr� rR Z _inheritingF) r r ri r" r rv r# r_ r] �pop)rF r� rc r6 r� rR r r r r� b s r� c C sr t |�}y || |f S tk r( Y nX i }x |D ]}td||f |� q4W t| tf|�}||_||| |f< |S )zB Return a proxy type whose methods are given by `exposed` zLdef %s(self, *args, **kwds): return self._callmethod(%r, args, kwds))r� r� �execr r r� )r r� �_cacheZdic�meth� ProxyTyper r r � MakeProxyTypew s r Tc C s� t | d }|dkrB|| j|d�}zt|dd| f�}W d|j� X |dkrX|dk rX|j}|dkrjtj� j}td| j |�}|| ||||d�} d| _ | S )z* Return an auto-proxy for `token` r N)rZ rN z AutoProxy[%s])r� rZ rR T) r[ r" r9 r� r� r ri rZ r r! r� ) r� rc r� rZ r� rR r� r� r r� r r r r� � s r� c @ s e Zd Zdd� Zdd� ZdS )� Namespacec K s | j j|� d S )N)r� �update)r$ r6 r r r r% � s zNamespace.__init__c C s^ t | jj� �}g }x,|D ]$\}}|jd�s|jd||f � qW |j� d| jjdj|�f S )NrH z%s=%rz%s(%s)z, ) r r� r � startswithrD r� r) r* r� )r$ r rE r r� r r r r+ � s zNamespace.__repr__N)r* r, r- r% r+ r r r r r � s r c @ s8 e Zd Zd dd�Zdd� Zdd� Zdd � Zeee�Zd S ) �ValueTc C s || _ || _d S )N)� _typecode�_value)r$ r r� �lockr r r r% � s zValue.__init__c C s | j S )N)r )r$ r r r r� � s z Value.getc C s || _ d S )N)r )r$ r� r r r r� � s z Value.setc C s dt | �j| j| jf S )Nz %s(%r, %r))r r* r r )r$ r r r r+ � s zValue.__repr__N)T) r* r, r- r% r� r� r+ r� r� r r r r r � s r c C s t j | |�S )N)r )r Zsequencer r r r �Array� s r c @ s8 e Zd ZdZdd� Zdd� Zd d � Zd d � Zd d� ZdS )� IteratorProxy�__next__r0 �throwr� c C s | S )Nr )r$ r r r �__iter__� s zIteratorProxy.__iter__c G s | j d|�S )Nr )r� )r$ r5 r r r r � s zIteratorProxy.__next__c G s | j d|�S )Nr0 )r� )r$ r5 r r r r0 � s zIteratorProxy.sendc G s | j d|�S )Nr )r� )r$ r5 r r r r � s zIteratorProxy.throwc G s | j d|�S )Nr� )r� )r$ r5 r r r r� � s zIteratorProxy.closeN)r r0 r r� ) r* r, r- r� r r r0 r r� r r r r r � s r c @ s2 e Zd Zd Zddd�Zdd� Zd d � Zd d � ZdS )� AcquirerProxy�acquire�releaseTNc C s" |d kr|fn||f}| j d|�S )Nr )r� )r$ Zblockingr� r5 r r r r � s zAcquirerProxy.acquirec C s | j d�S )Nr )r� )r$ r r r r � s zAcquirerProxy.releasec C s | j d�S )Nr )r� )r$ r r r r� � s zAcquirerProxy.__enter__c C s | j d�S )Nr )r� )r$ r� r� r� r r r r� � s zAcquirerProxy.__exit__)r r )TN)r* r, r- r� r r r� r� r r r r r � s r c @ s4 e Zd ZdZddd�Zd d � Zd d � Zdd d�ZdS )�ConditionProxyr r rp �notify� notify_allNc C s | j d|f�S )Nrp )r� )r$ r� r r r rp � s zConditionProxy.waitc C s | j d�S )Nr )r� )r$ r r r r � s zConditionProxy.notifyc C s | j d�S )Nr )r� )r$ r r r r � s zConditionProxy.notify_allc C sh |� }|r|S |d k r$t j� | }nd }d }x6|sb|d k rP|t j� }|dkrPP | j|� |� }q.W |S )Nr )�timeZ monotonicrp )r$ Z predicater� r8 ZendtimeZwaittimer r r �wait_for� s zConditionProxy.wait_for)r r rp r r )N)N)r* r, r- r� rp r r r r r r r r � s r c @ s2 e Zd ZdZdd� Zdd� Zd d � Zdd d �Zd S )� EventProxyro r� r� rp c C s | j d�S )Nro )r� )r$ r r r ro s zEventProxy.is_setc C s | j d�S )Nr� )r� )r$ r r r r� s zEventProxy.setc C s | j d�S )Nr� )r� )r$ r r r r� s zEventProxy.clearNc C s | j d|f�S )Nrp )r� )r$ r� r r r rp s zEventProxy.wait)ro r� r� rp )N)r* r, r- r� ro r� r� rp r r r r r � s r c @ sN e Zd ZdZddd�Zdd � Zd d � Zed d � �Zedd� �Z edd� �Z dS )� BarrierProxy�__getattribute__rp �abort�resetNc C s | j d|f�S )Nrp )r� )r$ r� r r r rp s zBarrierProxy.waitc C s | j d�S )Nr )r� )r$ r r r r s zBarrierProxy.abortc C s | j d�S )Nr )r� )r$ r r r r s zBarrierProxy.resetc C s | j dd�S )Nr �parties)r )r� )r$ r r r r s zBarrierProxy.partiesc C s | j dd�S )Nr � n_waiting)r )r� )r$ r r r r s zBarrierProxy.n_waitingc C s | j dd�S )Nr �broken)r )r� )r$ r r r r s zBarrierProxy.broken)r rp r r )N) r* r, r- r� rp r r r� r r r r r r r r s r c @ s( e Zd Zd Zdd� Zdd� Zdd � Zd S ) �NamespaceProxyr � __setattr__� __delattr__c C s0 |d dkrt j| |�S t j| d�}|d|f�S )Nr rH r� r )�objectr )r$ r� � callmethodr r r � __getattr__ s zNamespaceProxy.__getattr__c C s4 |d dkrt j| ||�S t j| d�}|d||f�S )Nr rH r� r )r! r r )r$ r� r� r" r r r r % s zNamespaceProxy.__setattr__c C s0 |d dkrt j| |�S t j| d�}|d|f�S )Nr rH r� r )r! r r )r$ r� r" r r r r * s zNamespaceProxy.__delattr__N)r r r )r* r, r- r� r# r r r r r r r s r c @ s* e Zd ZdZdd� Zdd� Zeee�ZdS ) � ValueProxyr� r� c C s | j d�S )Nr� )r� )r$ r r r r� 3 s zValueProxy.getc C s | j d|f�S )Nr� )r� )r$ r� r r r r� 5 s zValueProxy.setN)r� r� )r* r, r- r� r� r� r� r� r r r r r$ 1 s r$ � BaseListProxy�__add__� __contains__� __delitem__� __getitem__�__len__�__mul__� __reversed__�__rmul__� __setitem__rD �count�extend�index�insertr� �remove�reverser� �__imul__c @ s e Zd Zdd� Zdd� ZdS )� ListProxyc C s | j d|f� | S )Nr0 )r� )r$ r� r r r �__iadd__A s zListProxy.__iadd__c C s | j d|f� | S )Nr5 )r� )r$ r� r r r r5 D s zListProxy.__imul__N)r* r, r- r7 r5 r r r r r6 @ s r6 � DictProxyr r� r� r� �has_key�popitem� setdefaultr �Iterator� ArrayProxy� PoolProxy�apply� apply_asyncr� �imap�imap_unorderedr� �map� map_async�starmap� starmap_asyncr� Z AsyncResult)r@ rD rF rA rB c @ s e Zd Zdd� Zdd� ZdS )r> c C s | S )Nr )r$ r r r r� d s zPoolProxy.__enter__c C s | j � d S )N)r� )r$ r� r� r� r r r r� f s zPoolProxy.__exit__N)r* r, r- r� r� r r r r r> c s c @ s e Zd ZdZdS )r a( Subclass of `BaseManager` which supports a number of shared object types. The types registered are those intended for the synchronization of threads, plus `dict`, `list` and `Namespace`. The `multiprocessing.Manager()` function creates started instances of this class. N)r* r, r- r. r r r r r m s �QueueZ JoinableQueuerg ra �RLock� Semaphore�BoundedSemaphore� Condition�Barrier�Poolr r� F)r� r� )r� )r r r )NNNT)T)r& r' r( r) r* r+ r, r- r. rD r/ r0 r1 r2 r� r3 r4 r� r5 )r' r( r) r r* r. r� r� r� r9 r r r� r: r; r r )r* r) r. ) r? r@ r� rA rB r� rC rD rE rF r� )I�__all__rs r` r Zqueuer r� r � r �contextr r r r r r r r� Z view_typesr r Z view_typer! r r9 r2 r r< rG rI rJ r� rd re Z XmlListenerZ XmlClientr[ r r� r� r r� r r� r r r r r r r r r r$ r% r6 r8 r� r= Z BasePoolProxyr> r rG rg ra rH rI rJ rK rL rM r� r r r r �<module> s� ; ] 4 __pycache__/popen_forkserver.cpython-36.opt-1.pyc 0000644 00000004430 15204243005 0015726 0 ustar 00 3 \� � @ s� d dl Z d dlZddlmZmZ ejs.ed��ddlmZ ddlm Z ddlm Z ddlm Z d gZ G d d � d e �ZG d d � d e j�ZdS ) � N� )� reduction�set_spawning_popenz,No support for sending fds between processes)� forkserver)� popen_fork)�spawn)�util�Popenc @ s e Zd Zdd� Zdd� ZdS )�_DupFdc C s || _ d S )N)�ind)�selfr � r �8/usr/lib64/python3.6/multiprocessing/popen_forkserver.py�__init__ s z_DupFd.__init__c C s t j� | j S )N)r Zget_inherited_fdsr )r r r r �detach s z _DupFd.detachN)�__name__� __module__� __qualname__r r r r r r r s r c sB e Zd ZdZeZ� fdd�Zdd� Zdd� Ze j fdd �Z � Z S ) r r c s g | _ t� j|� d S )N)�_fds�superr )r � process_obj)� __class__r r r ! s zPopen.__init__c C s | j j|� t| j �d S )Nr )r �append�len)r �fdr r r �duplicate_for_child% s zPopen.duplicate_for_childc C s� t j|j�}tj� }t| � z tj||� tj||� W d td � X tj | j �\| _ }t j | tj| j f� t|ddd��}|j|j� � W d Q R X tj| j �| _d S )N�wbT)�closefd)r Zget_preparation_data�_name�io�BytesIOr r �dumpr Zconnect_to_new_processr �sentinelr ZFinalize�os�close�open�write� getbuffer� read_unsigned�pid)r r Z prep_dataZbuf�w�fr r r �_launch) s z Popen._launchc C sr | j d krlddlm} |tjkr$dnd }|| jg|�s:d S ytj| j�| _ W n tt fk rj d| _ Y nX | j S )Nr )�wait� ) � returncodeZmultiprocessing.connectionr- r# �WNOHANGr" r r( �OSError�EOFError)r �flagr- Ztimeoutr r r �poll9 s z Popen.poll) r r r �methodr ZDupFdr r r, r# r0 r4 � __classcell__r r )r r r s )r r# �contextr r ZHAVE_SEND_HANDLE� ImportError� r r r r �__all__�objectr r r r r r �<module> s __pycache__/queues.cpython-36.opt-1.pyc 0000644 00000021242 15204243005 0013644 0 ustar 00 3 \ * � @ s� d ddgZ ddlZddlZddlZddlZddlZddlZddlZddlm Z m Z ddl Z ddl m Z ddl mZ ejjZdd lmZmZmZmZmZ G d d � d e�Ze� ZG d d� de�ZG d d� de�Z dS ) �Queue� SimpleQueue� JoinableQueue� N)�Empty�Full� )� connection)�context)�debug�info�Finalize�register_after_fork� is_exitingc @ s� e Zd Zd(dd�Zdd� Zdd� Zdd � Zd)d d �Zd*dd�Zdd� Z dd� Z dd� Z dd� Z dd� Z dd� Zd d � Zd d � Zd d!� Zed"d#� �Zed$d%� �Zed&d'� �Zd S )+r r c C s� |dkrddl m} || _tjdd�\| _| _|j� | _t j � | _ t j dkrTd | _n |j� | _|j|�| _d| _| j� t j dkr�t| tj� d S )Nr r )� SEM_VALUE_MAXF)�duplex�win32)Z synchronizer �_maxsizer �Pipe�_reader�_writer�Lock�_rlock�os�getpid�_opid�sys�platform�_wlockZBoundedSemaphore�_sem� _ignore_epipe� _after_forkr r )�self�maxsize�ctx� r$ �./usr/lib64/python3.6/multiprocessing/queues.py�__init__$ s zQueue.__init__c C s. t j| � | j| j| j| j| j| j| j| j fS )N) r �assert_spawningr r r r r r r r )r! r$ r$ r% � __getstate__9 s zQueue.__getstate__c C s0 |\| _ | _| _| _| _| _| _| _| j� d S )N) r r r r r r r r r )r! �stater$ r$ r% � __setstate__> s $zQueue.__setstate__c C sb t d� tjtj� �| _tj� | _d | _d | _ d| _ d| _ d | _ | j j| _| jj| _| jj| _d S )NzQueue._after_fork()F)r � threading� Conditionr � _notempty� collections�deque�_buffer�_thread� _jointhread�_joincancelled�_closed�_closer � send_bytes� _send_bytesr � recv_bytes� _recv_bytes�poll�_poll)r! r$ r$ r% r C s zQueue._after_forkTNc C sP | j j||�st�| j�. | jd kr,| j� | jj|� | jj� W d Q R X d S )N) r �acquirer r- r1 � _start_threadr0 �append�notify)r! �obj�block�timeoutr$ r$ r% �putP s z Queue.putc C s� |r2|d kr2| j � | j� }W d Q R X | jj� nr|rBtj� | }| j j||�sTt�zB|rv|tj� }| j|�s�t�n | j� s�t�| j� }| jj� W d | j j� X t j |�S )N) r r9 r �release�timeZ monotonicr< r r; �_ForkingPickler�loads)r! rA rB �resZdeadliner$ r$ r% �get[ s& z Queue.getc C s | j | jjj� S )N)r r �_semlockZ _get_value)r! r$ r$ r% �qsizes s z Queue.qsizec C s | j � S )N)r; )r! r$ r$ r% �emptyw s z Queue.emptyc C s | j jj� S )N)r rJ �_is_zero)r! r$ r$ r% �fullz s z Queue.fullc C s | j d�S )NF)rI )r! r$ r$ r% � get_nowait} s zQueue.get_nowaitc C s | j |d�S )NF)rC )r! r@ r$ r$ r% � put_nowait� s zQueue.put_nowaitc C s2 d| _ z| jj� W d | j}|r,d | _|� X d S )NT)r4 r �closer5 )r! rQ r$ r$ r% rQ � s z Queue.closec C s t d� | jr| j� d S )NzQueue.join_thread())r r2 )r! r$ r$ r% � join_thread� s zQueue.join_threadc C s6 t d� d| _y| jj� W n tk r0 Y nX d S )NzQueue.cancel_join_thread()T)r r3 r2 Zcancel�AttributeError)r! r$ r$ r% �cancel_join_thread� s zQueue.cancel_join_threadc C s� t d� | jj� tjtj| j| j| j| j | j j | j fdd�| _ d| j _t d� | j j� t d� | js�t| j tjtj| j �gd d�| _t| tj| j| jgd d�| _d S ) NzQueue._start_thread()ZQueueFeederThread)�target�args�nameTzdoing self._thread.start()z ... done self._thread.start()� )Z exitpriority� ���)r r0 �clearr+ ZThreadr �_feedr- r7 r r rQ r r1 Zdaemon�startr3 r �_finalize_join�weakref�refr2 �_finalize_closer5 )r! r$ r$ r% r= � s( zQueue._start_threadc C s4 t d� | � }|d k r(|j� t d� nt d� d S )Nzjoining queue threadz... queue thread joinedz ... queue thread already dead)r �join)ZtwrZthreadr$ r$ r% r^ � s zQueue._finalize_joinc C s. t d� |� | jt� |j� W d Q R X d S )Nz telling queue thread to quit)r r> � _sentinelr? )�buffer�notemptyr$ r$ r% ra � s zQueue._finalize_closec C sP t d� |j}|j}|j}| j} t} tjdkr<|j} |j} nd } �xy�|� z| sX|� W d |� X ybx\| � } | | kr�t d� |� d S tj | �} | d kr�|| � qh| � z || � W d | � X qhW W n t k r� Y nX W qD t k �rF } zJ|�rt |dd�t jk�rd S t� �r&td|� d S dd l}|j� W Y d d }~X qDX qDW d S )Nz$starting thread to feed data to piper z%feeder thread got sentinel -- exiting�errnor zerror in queue thread: %s)r r< rD �wait�popleftrc r r rF �dumps� IndexError� Exception�getattrrf ZEPIPEr r � traceback� print_exc)rd re r6 Z writelockrQ Z ignore_epipeZnacquireZnreleaseZnwaitZbpopleft�sentinelZwacquireZwreleaser@ �erm r$ r$ r% r\ � sR z Queue._feed)r )TN)TN)�__name__� __module__� __qualname__r& r( r* r rC rI rK rL rN rO rP rQ rR rT r= � staticmethodr^ ra r\ r$ r$ r$ r% r " s$ c @ s@ e Zd Zddd�Zdd� Zdd� Zdd d �Zd d � Zdd� Zd S )r r c C s* t j| ||d� |jd�| _|j� | _d S )N)r# r )r r& Z Semaphore�_unfinished_tasksr, �_cond)r! r" r# r$ r$ r% r& s zJoinableQueue.__init__c C s t j| �| j| jf S )N)r r( rv ru )r! r$ r$ r% r( s zJoinableQueue.__getstate__c C s, t j| |d d� � |dd � \| _| _d S )N� ���rx )r r* rv ru )r! r) r$ r$ r% r* s zJoinableQueue.__setstate__TNc C sl | j j||�st�| j�J | j�8 | jd kr4| j� | jj|� | j j � | jj � W d Q R X W d Q R X d S )N) r r< r r- rv r1 r= r0 r> ru rD r? )r! r@ rA rB r$ r$ r% rC s zJoinableQueue.putc C s@ | j �0 | jjd�s td��| jjj� r2| j j� W d Q R X d S )NFz!task_done() called too many times)rv ru r< � ValueErrorrJ rM Z notify_all)r! r$ r$ r% � task_done' s zJoinableQueue.task_donec C s, | j � | jjj� s | j j� W d Q R X d S )N)rv ru rJ rM rg )r! r$ r$ r% rb . s zJoinableQueue.join)r )TN) rq rr rs r& r( r* rC rz rb r$ r$ r$ r% r s c @ s< e Zd Zdd� Zdd� Zdd� Zdd� Zd d � Zd d � Zd S )r c C sH t jdd�\| _| _|j� | _| jj| _tj dkr:d | _ n |j� | _ d S )NF)r r ) r r r r r r r: r; r r r )r! r# r$ r$ r% r& 9 s zSimpleQueue.__init__c C s | j � S )N)r; )r! r$ r$ r% rL B s zSimpleQueue.emptyc C s t j| � | j| j| j| jfS )N)r r' r r r r )r! r$ r$ r% r( E s zSimpleQueue.__getstate__c C s" |\| _ | _| _| _| j j| _d S )N)r r r r r: r; )r! r) r$ r$ r% r* I s zSimpleQueue.__setstate__c C s& | j � | jj� }W d Q R X tj|�S )N)r r r8 rF rG )r! rH r$ r$ r% rI M s zSimpleQueue.getc C sD t j|�}| jd kr"| jj|� n | j� | jj|� W d Q R X d S )N)rF ri r r r6 )r! r@ r$ r$ r% rC S s zSimpleQueue.putN) rq rr rs r& rL r( r* rI rC r$ r$ r$ r% r 7 s ) �__all__r r r+ r. rE r_ rf Zqueuer r Z_multiprocessing� r r Z reductionZForkingPicklerrF �utilr r r r r �objectr rc r r r$ r$ r$ r% �<module> s$ b * __pycache__/forkserver.cpython-36.opt-2.pyc 0000644 00000013501 15204243005 0014525 0 ustar 00 3 \�! � @ s� d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZddlm Z ddlm Z ddl m Z ddlm Z ddlmZ ddlmZ d d d d gZd Zejd�ZG dd� de�Zddd�Zdd� Zdd� Zdd� Ze� ZejZej Z ej Z ej Z dS )� N� )� connection)�process)� reduction)�semaphore_tracker)�spawn)�util�ensure_running�get_inherited_fds�connect_to_new_process�set_forkserver_preload� �Qc @ s4 e Zd Zdd� Zdd� Zdd� Zdd� Zd d � Zd S ) � ForkServerc C s. d | _ d | _d | _d | _tj� | _dg| _d S )N�__main__)�_forkserver_address�_forkserver_alive_fd�_forkserver_pid�_inherited_fds� threadingZLock�_lock�_preload_modules)�self� r �2/usr/lib64/python3.6/multiprocessing/forkserver.py�__init__! s zForkServer.__init__c C s& t dd� | jD ��s td��|| _d S )Nc s s | ]}t |�tkV qd S )N)�type�str)�.0�modr r r � <genexpr>+ s z4ForkServer.set_forkserver_preload.<locals>.<genexpr>z&module_names must be a list of strings)�allr � TypeError)r Z modules_namesr r r r ) s z!ForkServer.set_forkserver_preloadc C s | j S )N)r )r r r r r / s z ForkServer.get_inherited_fdsc C s� | j � t|�d tkr td��tjtj���}|j| j� tj � \}}tj � \}}||| j t j � g}||7 }z<yt j||� ||fS tj|� tj|� � Y nX W d tj|� tj|� X W d Q R X d S )N� z too many fds)r �len�MAXFDS_TO_SEND� ValueError�socket�AF_UNIXZconnectr �os�piper r Zgetfdr Zsendfds�close)r �fdsZclientZparent_r�child_w�child_rZparent_wZallfdsr r r r 7 s& z!ForkServer.connect_to_new_processc s� | j ��n tj� | jd k rVtj| jtj�\}}|s8d S tj| j� d | _ d | _d | _d }| j r�ddh� t j d�}t � fdd�|j� D ��}ni }tjtj���}tjd�}|j|� tj|d � |j� tj� \}}ztyV|j� |g} ||j� || j |f; }t j� } | gtj� } | d |g7 } tj| | | �}W n tj|� � Y nX W d tj|� X || _ || _|| _W d Q R X W d Q R X d S ) Nz-from multiprocessing.forkserver import main; zmain(%d, %d, %r, **%r)� main_path�sys_path�ignorec 3 s" | ]\}}|� kr||fV qd S )Nr )r �x�y)� desired_keysr r r o s z,ForkServer.ensure_running.<locals>.<genexpr>r( i� z-czCfrom multiprocessing.forkserver import main; main(%d, %d, %r, **%r)) r r r r r) �waitpid�WNOHANGr+ r r r r Zget_preparation_data�dict�itemsr' r( r Zarbitrary_addressZbind�chmodZlistenr* �filenoZget_executabler Z _args_from_interpreter_flagsZspawnv_passfds) r �pidZstatus�cmd�data�listenerZaddress�alive_rZalive_wZ fds_to_passZexe�argsr )r4 r r T sL zForkServer.ensure_runningN)�__name__� __module__� __qualname__r r r r r r r r r r s r c 4 C s� |rhd|kr8|d k r8dt j� _ztj|� W d t j� `X x.|D ]&}y t|� W q> tk rb Y q>X q>W tj� t j t j t j t j i}dd� |j � D �}tjtj| d���,}tj� ��} |j� t_| j|tj� | j|tj� x�y�x dd� | j� D �} | r�P q�W || k�r t�|j� d �l} d } tj� dk�r�zFyt | |||� W n. t k �rn t j t j � � t j!j"� Y nX W d tj#| � X W d Q R X W q� t$k �r� } z| j%t%j&k�r�� W Y d d } ~ X q�X q�W W d Q R X W d Q R X d S ) Nr Tc S s i | ]\}}t j ||�|�qS r )�signal)r �sig�valr r r � <dictcomp>� s zmain.<locals>.<dictcomp>)r: c S s g | ]\}}|j �qS r )Zfileobj)r �keyZeventsr r r � <listcomp>� s zmain.<locals>.<listcomp>r r )'r Zcurrent_processZ _inheritingr Zimport_main_path� __import__� ImportErrorr Z _close_stdinrD �SIGCHLD�SIG_IGN�SIGINTr8 r' r( � selectorsZDefaultSelectorZ getsockname� _forkserverr �registerZ EVENT_READZselect� SystemExitZacceptr) �fork� _serve_one� Exception�sys� excepthook�exc_info�stderr�flush�_exit�OSError�errnoZ ECONNABORTED)Z listener_fdr? Zpreloadr/ r0 �modname�handlersZ old_handlersr> ZselectorZrfds�s�code�er r r �main� sR rc c C s� |j � tj |� x |j� D ]\}}tj||� q W tj| td �}| j � |^}}t_} t_ | t j _ t |tj� � dtjkr�dd l} | j� tj|�} t || � d S )Nr �randomr )r+ r) r8 rD r Zrecvfdsr% rP r r r Z_semaphore_trackerZ_fd�write_unsigned�getpidrV �modulesrd Zseedr �_main) r` r> r? r_ rE rF r, r. r- Zstfdrd ra r r r rT � s rT c C sR d}t j}x8t|�|k rBtj| |t|� �}|s8td��||7 }q W t j|�d S )N� zunexpected EOFr )�UNSIGNED_STRUCT�sizer$ r) �read�EOFErrorZunpack)�fdr= Zlengthr` r r r � read_unsigned� s ro c C s@ t j|�}x0|r:tj| |�}|dkr,td��||d � }q W d S )Nr zshould not get here)rj Zpackr) �write� RuntimeError)rn �n�msg�nbytesr r r re � s re )NN) r] r) rO rD r' ZstructrV r � r r �contextr r r r �__all__r% ZStructrj �objectr rc rT ro re rP r r r r r r r r �<module> s6 s =! __pycache__/managers.cpython-36.opt-1.pyc 0000644 00000100054 15204243005 0014131 0 ustar 00 3 \� � @ s� d dddgZ ddlZddlZddlZddlZddlZddlmZ ddlm Z dd l m Z m Z dd lm Z dd lmZ dd lmZ dd lmZ dd� Ze jeje� dd� d�D �Zed ek r�dd� ZxeD ]Ze jee� q�W G dd� de�Zf i fdd�Zdd� ZG d d � d e�Z d d � Z d d!� Z G d"d#� d#e�Z G d$d%� d%e�Z e j!e j"fe j#e j$fd&�Z%G d'd � d e�Z&G d(d)� d)e'�Z(G d*d� de�Z)d+d,� Z*i fd-d.�Z+d�d0d1�Z,G d2d3� d3e�Z-G d4d5� d5e�Z.d�d6d7�Z/G d8d9� d9e)�Z0G d:d;� d;e)�Z1G d<d=� d=e1�Z2G d>d?� d?e)�Z3G d@dA� dAe)�Z4G dBdC� dCe)�Z5G dDdE� dEe)�Z6e+dFd��Z7G dZd[� d[e7�Z8e+d\d��Z9d]deie9_:e+dfd��Z;e+dgd��Z<dsdsdsdededt�e<_:G dudg� dge<�Z=G dvd� de&�Z>e>jdwej?� e>jdxej?� e>jdyej@e3� e>jdzejAe1� e>jd{ejBe1� e>jd|ejCe1� e>jd}ejDe1� e>jd~ejEe2� e>jdejFe4� e>jd�e jGe=� e>jd�ee8� e>jd�eHe9� e>jd5e.e6� e>jd7e/e;� e>jd3e-e5� e>jdee0d�d�� e>jdsd�d�� dS )�� BaseManager� SyncManager� BaseProxy�Token� N)� format_exc� )� connection)� reduction�get_spawning_popen)�pool)�process)�util)� get_contextc C s t j | j| j� ffS )N)�array�typecode�tobytes)�a� r �0/usr/lib64/python3.6/multiprocessing/managers.py� reduce_array$ s r c C s g | ]}t ti |�� ��qS r )�type�getattr)�.0�namer r r � <listcomp>( s r �items�keys�valuesc C s t t | �ffS )N)�list)�objr r r �rebuild_as_list* s r c @ s4 e Zd ZdZdZdd� Zdd� Zd d � Zd d � Zd S )r z4 Type to uniquely indentify a shared object �typeid�address�idc C s ||| | _ | _| _d S )N)r! r" r# )�selfr! r" r# r r r �__init__9 s zToken.__init__c C s | j | j| jfS )N)r! r" r# )r$ r r r � __getstate__< s zToken.__getstate__c C s |\| _ | _| _d S )N)r! r" r# )r$ �stater r r � __setstate__? s zToken.__setstate__c C s d| j j| j| j| jf S )Nz %s(typeid=%r, address=%r, id=%r))� __class__�__name__r! r" r# )r$ r r r �__repr__B s zToken.__repr__N)r! r" r# ) r* � __module__� __qualname__�__doc__� __slots__r% r& r( r+ r r r r r 3 s c C s8 | j ||||f� | j� \}}|dkr*|S t||��dS )zL Send a message to manager using connection `c` and return response z#RETURNN)�send�recv�convert_to_error)�cr# � methodname�args�kwds�kind�resultr r r �dispatchJ s r9 c C s< | dkr |S | dkr t |�S | dkr0t d| �S td�S d S )Nz#ERRORz #TRACEBACKz#UNSERIALIZABLEzUnserializable message: %s zUnrecognized message type)� RemoteError� ValueError)r7 r8 r r r r2 T s r2 c @ s e Zd Zdd� ZdS )r: c C s& ddd d t | jd � dd S )N� �-�K r )�strr5 )r$ r r r �__str__a s zRemoteError.__str__N)r* r, r- r@ r r r r r: ` s r: c C s6 g }x,t | �D ] }t| |�}t|�r|j|� qW |S )z4 Return a list of names of methods of `obj` )�dirr �callable�append)r �tempr �funcr r r � all_methodsh s rF c C s dd� t | �D �S )zP Return a list of names of methods of `obj` which do not start with '_' c S s g | ]}|d dkr|�qS )r �_r )r r r r r r w s z"public_methods.<locals>.<listcomp>)rF )r r r r �public_methodss s rH c @ s� e Zd ZdZdddddddd d g Zd d � Zd d� Zdd� Zdd� Zdd� Z dd� Z dd� Z dd� Z e e e d�Z d d � Zd d � Zd d!� Zd"d#� Zd$d%� Zd&d'� Zd(d)� Zd*d+� Zd,d-� Zd.S )/�ServerzM Server class which runs in a process controlled by a manager object �shutdown�create�accept_connection� get_methods� debug_info�number_of_objects�dummy�incref�decrefc C s^ || _ tj|�| _t| \}}||dd�| _| jj| _dd f fi| _i | _i | _ t j � | _ d S )N� )r" Zbacklog�0) �registryr �AuthenticationString�authkey�listener_client�listenerr" � id_to_obj�id_to_refcount�id_to_local_proxy_obj� threading�Lock�mutex)r$ rU r" rW � serializer�Listener�Clientr r r r% � s zServer.__init__c C s� t j� | _| tj� _zZt j| jd�}d|_|j � y x| jj � sN| jj d� q6W W n t t fk rj Y nX W dtjtjkr�tjd� tjt_tjt_tjd� X dS )z( Run the server forever )�targetTr Nzresetting stdout, stderrr )r] �Event� stop_eventr �current_process�_manager_server�Thread�accepter�daemon�start�is_set�wait�KeyboardInterrupt� SystemExit�sys�stdout� __stdout__r �debug� __stderr__�stderr�exit)r$ ri r r r � serve_forever� s zServer.serve_foreverc C sP xJy| j j� }W n tk r& wY nX tj| j|fd�}d|_|j� qW d S )N)rc r5 T)rY Zaccept�OSErrorr] rh �handle_requestrj rk )r$ r3 �tr r r ri � s zServer.accepterc ' C s4 d } }}y>t j|| j� t j|| j� |j� }|\}}}}t| |�}W n tk rh dt� f} Y n>X y||f|�|�}W n tk r� dt� f} Y n X d|f} y|j| � W nr tk �r& } zTy|jdt� f� W n tk r� Y nX t j d| � t j d|� t j d| � W Y dd} ~ X nX |j � dS )z) Handle a new connection Nz #TRACEBACKz#RETURNzFailure to send message: %rz ... request was %rz ... exception was %r) r Zdeliver_challengerW Zanswer_challenger1 r � Exceptionr r0 r �info�close) r$ r3 �funcnamer8 �request�ignorer5 r6 rE �msg�er r r ry � s2 zServer.handle_requestc D C s� t jdtj� j� |j}|j}| j}�x�| jj � �s�yBd }}|� }|\}}} } y|| \}} } W n^ t k r� } zBy| j | \}} } W n& t k r� } z | �W Y dd}~X nX W Y dd} ~ X nX || kr�t d|t |�| f ��t||�}y|| | �}W n, tk �r& } zd|f}W Y dd}~X nPX | �o8| j|d�}|�rn| j|||�\}}t|| j|�}d||ff}nd|f}W n� t k �r� |dk�r�dt� f}nNy,| j| }|| |||f| �| �}d|f}W n tk �r� dt� f}Y nX Y nP tk �r" t jdtj� j� tjd � Y n tk �r@ dt� f}Y nX yDy ||� W n2 tk �r� } z|d t� f� W Y dd}~X nX W q( tk �r� } zBt jd tj� j� t jd |� t jd |� |j� tjd� W Y dd}~X q(X q(W dS )zQ Handle requests from the proxies in a particular process/thread z$starting server thread to service %rNz+method %r of %r object is not in exposed=%rz#ERRORz#PROXYz#RETURNz #TRACEBACKz$got EOF -- exiting thread serving %rr z#UNSERIALIZABLEz exception in thread serving %rz ... message was %rz ... exception was %rr )r rs r] �current_threadr r1 r0 rZ re rl �KeyErrorr\ �AttributeErrorr r r{ �getrK r r" r �fallback_mapping�EOFErrorrp rv r| r} )r$ �connr1 r0 rZ r4 r r �identr5 r6 �exposed� gettypeid�keZ second_keZfunction�resr� r� r! ZridentZrexposed�tokenZ fallback_funcr8 r r r � serve_client� sx ( $ zServer.serve_clientc C s |S )Nr )r$ r� r� r r r r �fallback_getvalue& s zServer.fallback_getvaluec C s t |�S )N)r? )r$ r� r� r r r r � fallback_str) s zServer.fallback_strc C s t |�S )N)�repr)r$ r� r� r r r r � fallback_repr, s zServer.fallback_repr)r@ r+ z #GETVALUEc C s d S )Nr )r$ r3 r r r rP 5 s z Server.dummyc C s| | j �l g }t| jj� �}|j� xD|D ]<}|dkr(|jd|| j| t| j| d �dd� f � q(W dj|�S Q R X dS )zO Return some info --- useful to spot problems with refcounting rT z %s: refcount=%s %sr Nr> r< ) r_ r r[ r �sortrC r? rZ �join)r$ r3 r8 r r� r r r rN 8 s $zServer.debug_infoc C s t | j�S )z* Number of shared objects )�lenr[ )r$ r3 r r r rO G s zServer.number_of_objectsc C sL z:yt jd� |jd� W n ddl}|j� Y nX W d| jj� X dS )z' Shutdown this process z!manager received shutdown message�#RETURNNr )r� N)r rs r0 � traceback� print_excre �set)r$ r3 r� r r r rJ N s zServer.shutdownc O s� | j �� | j| \}}}}|dkr,|d } n |||�} |dkrFt| �}|dk r^t|�t|� }dt| � } tjd|| � | t|�|f| j| < | | j kr�d| j | < W dQ R X | j || � | t |�fS )z> Create a new shared object and return its id Nr z%xz&%r callable returned object with id %r) r_ rU rH r r# r rs r� rZ r[ rQ �tuple) r$ r3 r! r5 r6 rB r� �method_to_typeid� proxytyper r� r r r rK [ s z Server.createc C s t | j|j d �S )zL Return the methods of the shared object indicated by token r )r� rZ r# )r$ r3 r� r r r rM z s zServer.get_methodsc C s" |t j� _|jd� | j|� dS )z= Spawn a new thread to serve this connection �#RETURNN)r� N)r] r� r r0 r� )r$ r3 r r r r rL � s zServer.accept_connectionc C s� | j �� y| j| d7 < W nh tk r� } zL|| jkrrd| j|< | j| | j|< | j| \}}}tjd|� n|�W Y d d }~X nX W d Q R X d S )Nr z&Server re-enabled tracking & INCREF %r)r_ r[ r� r\ rZ r rs )r$ r3 r� r� r r� r� r r r rQ � s z Server.increfc C s� || j kr$|| jkr$tjd|� d S | j�. | j | d8 < | j | dkrT| j |= W d Q R X || j kr�d f d f| j|< tjd|� | j� | j|= W d Q R X d S )NzServer DECREF skipping %rr r zdisposing of obj with id %r)r[ r\ r rs r_ rZ )r$ r3 r� r r r rR � s z Server.decrefN)r* r, r- r. Zpublicr% rw ri ry r� r� r� r� r� rP rN rO rJ rK rM rL rQ rR r r r r rI } s. "Q rI c @ s e Zd ZdgZdZdZdZdS )�State�valuer r � N)r* r, r- r/ �INITIAL�STARTED�SHUTDOWNr r r r r� � s r� )�pickleZ xmlrpclibc @ s� e Zd ZdZi ZeZd!dd�Zdd� Zdd � Z df fd d �Z e df fd d ��Z dd� Z d"dd�Zdd� Zdd� Zdd� Zdd� Zedd� �Zed d � �Ze d#d d ��ZdS )$r z! Base class for managers Nr� c C s\ |d krt j� j}|| _t j|�| _t� | _tj| j_ || _ t | \| _ | _ |pTt� | _d S )N)r rf rW �_addressrV �_authkeyr� �_stater� r� � _serializerrX Z _Listener�_Clientr �_ctx)r$ r" rW r` Zctxr r r r% � s zBaseManager.__init__c C s t | j| j| j| j�S )zX Return server object with serve_forever() method and address attribute )rI � _registryr� r� r� )r$ r r r � get_server� s zBaseManager.get_serverc C s8 t | j \}}|| j| jd�}t|dd� tj| j_dS )z> Connect manager object to the server process )rW NrP ) rX r� r� r� r9 r� r� r� r� )r$ ra rb r� r r r �connect� s zBaseManager.connectc C s� |dk rt |� rtd��tjdd�\}}| jjt| �j| j| j | j | j |||fd�| _ dj dd� | j jD ��}t| �jd | | j _| j j� |j� |j� | _ |j� tj| j_tj| t| �j| j | j | j | j| jfd d �| _ dS ) z@ Spawn a server process for this manager object Nz initializer must be a callableF)Zduplex)rc r5 �:c s s | ]}t |�V qd S )N)r? )r �ir r r � <genexpr>� s z$BaseManager.start.<locals>.<genexpr>r= r )r5 � exitpriority) rB � TypeErrorr ZPiper� ZProcessr � _run_serverr� r� r� r� �_processr� Z _identityr* r rk r} r1 r� r� r� r� r �Finalize�_finalize_managerr� rJ )r$ � initializer�initargs�reader�writerr� r r r rk � s& zBaseManager.startc C sN |dk r||� | j ||||�}|j|j� |j� tjd|j� |j� dS )z@ Create a server, report its address and run it Nzmanager serving at %r)�_Serverr0 r" r} r r| rw ) �clsrU r" rW r` r� r� r� �serverr r r r� s zBaseManager._run_serverc O sN | j | j| jd�}z t|dd|f| |�\}}W d|j� X t|| j|�|fS )zP Create a new shared object; return the token and exposed tuple )rW NrK )r� r� r� r9 r} r )r$ r! r5 r6 r� r# r� r r r �_create% s zBaseManager._createc C s* | j dk r&| j j|� | j j� s&d| _ dS )zC Join the manager process (if it has been spawned) N)r� r� �is_alive)r$ �timeoutr r r r� 1 s zBaseManager.joinc C s. | j | j| jd�}z t|dd�S |j� X dS )zS Return some info about the servers shared objects and connections )rW NrN )r� r� r� r9 r} )r$ r� r r r � _debug_info: s zBaseManager._debug_infoc C s. | j | j| jd�}z t|dd�S |j� X dS )z5 Return the number of shared objects )rW NrO )r� r� r� r9 r} )r$ r� r r r �_number_of_objectsD s z BaseManager._number_of_objectsc C s | j jtjkr| j� | S )N)r� r� r� r� rk )r$ r r r � __enter__N s zBaseManager.__enter__c C s | j � d S )N)rJ )r$ �exc_type�exc_val�exc_tbr r r �__exit__T s zBaseManager.__exit__c C s� | j � r�tjd� y,|||d�}zt|dd� W d|j� X W n tk rR Y nX | jdd� | j � r�tjd� t| d�r�tjd � | j� | jd d� | j � r�tjd � t j |_ y t j |= W n tk r� Y nX dS ) zQ Shutdown the manager process; will be registered as a finalizer z#sending shutdown message to manager)rW NrJ g �?)r� zmanager still alive� terminatez'trying to `terminate()` manager processg�������?z#manager still alive after terminate)r� r r| r9 r} r{ r� �hasattrr� r� r� r� r �_address_to_localr� )r r" rW r' r� r� r r r r� W s. z BaseManager._finalize_managerc C s | j S )N)r� )r$ r r r �<lambda>w s zBaseManager.<lambda>Tc s� d| j kr| jj� | _� dkr"t� |p0t� dd�}|p@t� dd�}|r`xt|j� �D ]\}}qTW |||� f| j�< |r�� �fdd�} �| _t| �| � dS )z9 Register a typeid with the manager type r� N� _exposed_�_method_to_typeid_c s` t jd�� | j�f|�|�\}}� || j| | j|d�}| j|j| jd�}t|d d|jf� |S )Nz)requesting creation of a shared %r object)�managerrW r� )rW rR ) r rs r� r� r� r� r" r9 r# )r$ r5 r6 r� Zexp�proxyr� )r� r! r r rD � s z"BaseManager.register.<locals>.temp) �__dict__r� �copy� AutoProxyr r r r* �setattr) r� r! rB r� r� r� � create_method�keyr� rD r )r� r! r �registery s zBaseManager.register)NNr� N)N)NNNNT)r* r, r- r. r� rI r� r% r� r� rk � classmethodr� r� r� r� r� r� r� � staticmethodr� �propertyr" r� r r r r r � s* $ c @ s e Zd Zdd� Zdd� ZdS )�ProcessLocalSetc C s t j| dd� � d S )Nc S s | j � S )N)�clear)r r r r r� � s z*ProcessLocalSet.__init__.<locals>.<lambda>)r �register_after_fork)r$ r r r r% � s zProcessLocalSet.__init__c C s t | �f fS )N)r )r$ r r r � __reduce__� s zProcessLocalSet.__reduce__N)r* r, r- r% r� r r r r r� � s r� c @ s� e Zd ZdZi Zej� Zddd�Zdd� Z f i fd d �Z d d � Z d d� Z e dd� �Zdd� Zdd� Zdd� Zdd� Zdd� ZdS ) r z. A base for proxies of shared objects NTFc C s� t j�8 t jj|jd �}|d kr:tj� t� f}|t j|j< W d Q R X |d | _|d | _ || _ | j j | _ || _ || _t| d | _|| _|d k r�tj|�| _n"| j d k r�| j j| _n tj� j| _|r�| j� tj| t j� d S )Nr r )r �_mutexr� r� r" r ZForkAwareLocalr� �_tls�_idset�_tokenr# �_id�_managerr� rX r� �_owned_by_managerr rV r� rf rW �_increfr� � _after_fork) r$ r� r` r� rW r� rQ � manager_ownedZ tls_idsetr r r r% � s* zBaseProxy.__init__c C sd t jd� tj� j}tj� jdkr4|dtj� j 7 }| j| jj | j d�}t |d d|f� || j _ d S )Nz making connection to managerZ MainThread�|)rW rL )r rs r rf r r] r� r� r� r" r� r9 r� r )r$ r r� r r r �_connect� s zBaseProxy._connectc C s� y | j j}W n6 tk rB tjdtj� j� | j� | j j}Y nX |j | j |||f� |j � \}}|dkrp|S |dkr�|\}}| j j |j d } | jj|_| || j| j | j|d�} | j|j| jd�}t|dd|jf� | S t||��dS ) zW Try to call a method of the referrent and return a copy of the result z#thread %r does not own a connectionz#RETURNz#PROXYr )r� rW r� )rW NrR ���)r� r r� r rs r] r� r r� r0 r� r1 r� r� r! r� r" r� r� r� r9 r# r2 ) r$ r4 r5 r6 r� r7 r8 r� r� r� r� r r r � _callmethod� s, zBaseProxy._callmethodc C s | j d�S )z9 Get a copy of the value of the referent z #GETVALUE)r� )r$ r r r � _getvalue s zBaseProxy._getvaluec C s� | j rtjd| jj� d S | j| jj| jd�}t|d d| j f� tjd| jj� | j j | j � | j oj| j j }tj| tj| j| j|| j| j | jfdd�| _d S )Nz%owned_by_manager skipped INCREF of %r)rW rQ z INCREF %r� )r5 r� )r� r rs r� r# r� r" r� r9 r� r� �addr� r� r� r �_decrefr� Z_close)r$ r� r' r r r r� s zBaseProxy._increfc C s� |j | j� |d ks |jtjkr�y2tjd| j� || j|d�}t|d d| jf� W q� t k r� } ztjd|� W Y d d }~X q�X ntjd| j� | r�t |d�r�tjdt j � j � |jj� |`d S )Nz DECREF %r)rW rR z... decref failed %sz%DECREF %r -- manager already shutdownr z-thread %r has no more proxies so closing conn)�discardr# r� r� r� r rs r" r9 r{ r� r] r� r r r} )r� rW r' ZtlsZidsetr� r� r� r r r r� s zBaseProxy._decrefc C sH d | _ y | j� W n0 tk rB } ztjd| � W Y d d }~X nX d S )Nzincref failed: %s)r� r� r{ r r| )r$ r� r r r r� 9 s zBaseProxy._after_forkc C s^ i }t � d k r| j|d< t| dd�rB| j|d< tt| j| j|ffS tt| �| j| j|ffS d S )NrW �_isautoFr� ) r r� r r� � RebuildProxyr� r� r� r )r$ r6 r r r r� A s zBaseProxy.__reduce__c C s | j � S )N)r� )r$ �memor r r � __deepcopy__N s zBaseProxy.__deepcopy__c C s dt | �j| jjt| �f S )Nz <%s object, typeid %r at %#x>)r r* r� r! r# )r$ r r r r+ Q s zBaseProxy.__repr__c C s4 y | j d�S tk r. t| �dd� d S X dS )zV Return representation of the referent (or a fall-back if that fails) r+ Nr z; '__str__()' failed>r� )r� r{ r� )r$ r r r r@ U s zBaseProxy.__str__)NNNTF)r* r, r- r. r� r ZForkAwareThreadLockr� r% r� r� r� r� r� r� r� r� r� r+ r@ r r r r r � s ( c C s� t tj� dd�}|rT|j|jkrTtjd|� d|d< |j|jkrT|j|j |j|j< |j dd�opt tj� dd� }| ||fd|i|��S ) z5 Function used for unpickling proxy objects. rg Nz*Rebuild a proxy owned by manager, token=%rTr� rQ Z _inheritingF) r r rf r" r rs r# r\ rZ �pop)rE r� r` r6 r� rQ r r r r� b s r� c C sr t |�}y || |f S tk r( Y nX i }x |D ]}td||f |� q4W t| tf|�}||_||| |f< |S )zB Return a proxy type whose methods are given by `exposed` zLdef %s(self, *args, **kwds): return self._callmethod(%r, args, kwds))r� r� �execr r r� )r r� �_cacheZdic�meth� ProxyTyper r r � MakeProxyTypew s r� Tc C s� t | d }|dkrB|| j|d�}zt|dd| f�}W d|j� X |dkrX|dk rX|j}|dkrjtj� j}td| j |�}|| ||||d�} d| _ | S )z* Return an auto-proxy for `token` r N)rW rM z AutoProxy[%s])r� rW rQ T) rX r" r9 r} r� r rf rW r� r! r� ) r� r` r� rW r� rQ r� r� r� r� r r r r� � s r� c @ s e Zd Zdd� Zdd� ZdS )� Namespacec K s | j j|� d S )N)r� �update)r$ r6 r r r r% � s zNamespace.__init__c C s^ t | jj� �}g }x,|D ]$\}}|jd�s|jd||f � qW |j� d| jjdj|�f S )NrG z%s=%rz%s(%s)z, ) r r� r � startswithrC r� r) r* r� )r$ r rD r r� r r r r+ � s zNamespace.__repr__N)r* r, r- r% r+ r r r r r� � s r� c @ s8 e Zd Zd dd�Zdd� Zdd� Zdd � Zeee�Zd S ) �ValueTc C s || _ || _d S )N)� _typecode�_value)r$ r r� �lockr r r r% � s zValue.__init__c C s | j S )N)r )r$ r r r r� � s z Value.getc C s || _ d S )N)r )r$ r� r r r r� � s z Value.setc C s dt | �j| j| jf S )Nz %s(%r, %r))r r* r r )r$ r r r r+ � s zValue.__repr__N)T) r* r, r- r% r� r� r+ r� r� r r r r r � s r c C s t j | |�S )N)r )r Zsequencer r r r �Array� s r c @ s8 e Zd ZdZdd� Zdd� Zd d � Zd d � Zd d� ZdS )� IteratorProxy�__next__r0 �throwr} c C s | S )Nr )r$ r r r �__iter__� s zIteratorProxy.__iter__c G s | j d|�S )Nr )r� )r$ r5 r r r r � s zIteratorProxy.__next__c G s | j d|�S )Nr0 )r� )r$ r5 r r r r0 � s zIteratorProxy.sendc G s | j d|�S )Nr )r� )r$ r5 r r r r � s zIteratorProxy.throwc G s | j d|�S )Nr} )r� )r$ r5 r r r r} � s zIteratorProxy.closeN)r r0 r r} ) r* r, r- r� r r r0 r r} r r r r r � s r c @ s2 e Zd Zd Zddd�Zdd� Zd d � Zd d � ZdS )� AcquirerProxy�acquire�releaseTNc C s" |d kr|fn||f}| j d|�S )Nr )r� )r$ Zblockingr� r5 r r r r � s zAcquirerProxy.acquirec C s | j d�S )Nr )r� )r$ r r r r � s zAcquirerProxy.releasec C s | j d�S )Nr )r� )r$ r r r r� � s zAcquirerProxy.__enter__c C s | j d�S )Nr )r� )r$ r� r� r� r r r r� � s zAcquirerProxy.__exit__)r r )TN)r* r, r- r� r r r� r� r r r r r � s r c @ s4 e Zd ZdZddd�Zd d � Zd d � Zdd d�ZdS )�ConditionProxyr r rm �notify� notify_allNc C s | j d|f�S )Nrm )r� )r$ r� r r r rm � s zConditionProxy.waitc C s | j d�S )Nr )r� )r$ r r r r � s zConditionProxy.notifyc C s | j d�S )Nr )r� )r$ r r r r � s zConditionProxy.notify_allc C sh |� }|r|S |d k r$t j� | }nd }d }x6|sb|d k rP|t j� }|dkrPP | j|� |� }q.W |S )Nr )�timeZ monotonicrm )r$ Z predicater� r8 ZendtimeZwaittimer r r �wait_for� s zConditionProxy.wait_for)r r rm r r )N)N)r* r, r- r� rm r r r r r r r r � s r c @ s2 e Zd ZdZdd� Zdd� Zd d � Zdd d �Zd S )� EventProxyrl r� r� rm c C s | j d�S )Nrl )r� )r$ r r r rl s zEventProxy.is_setc C s | j d�S )Nr� )r� )r$ r r r r� s zEventProxy.setc C s | j d�S )Nr� )r� )r$ r r r r� s zEventProxy.clearNc C s | j d|f�S )Nrm )r� )r$ r� r r r rm s zEventProxy.wait)rl r� r� rm )N)r* r, r- r� rl r� r� rm r r r r r � s r c @ sN e Zd ZdZddd�Zdd � Zd d � Zed d � �Zedd� �Z edd� �Z dS )� BarrierProxy�__getattribute__rm �abort�resetNc C s | j d|f�S )Nrm )r� )r$ r� r r r rm s zBarrierProxy.waitc C s | j d�S )Nr )r� )r$ r r r r s zBarrierProxy.abortc C s | j d�S )Nr )r� )r$ r r r r s zBarrierProxy.resetc C s | j dd�S )Nr �parties)r )r� )r$ r r r r s zBarrierProxy.partiesc C s | j dd�S )Nr � n_waiting)r )r� )r$ r r r r s zBarrierProxy.n_waitingc C s | j dd�S )Nr �broken)r )r� )r$ r r r r s zBarrierProxy.broken)r rm r r )N) r* r, r- r� rm r r r� r r r r r r r r s r c @ s( e Zd Zd Zdd� Zdd� Zdd � Zd S ) �NamespaceProxyr � __setattr__� __delattr__c C s0 |d dkrt j| |�S t j| d�}|d|f�S )Nr rG r� r )�objectr )r$ r� � callmethodr r r � __getattr__ s zNamespaceProxy.__getattr__c C s4 |d dkrt j| ||�S t j| d�}|d||f�S )Nr rG r� r )r r r )r$ r� r� r r r r r % s zNamespaceProxy.__setattr__c C s0 |d dkrt j| |�S t j| d�}|d|f�S )Nr rG r� r )r r r )r$ r� r r r r r * s zNamespaceProxy.__delattr__N)r r r )r* r, r- r� r r r r r r r r s r c @ s* e Zd ZdZdd� Zdd� Zeee�ZdS ) � ValueProxyr� r� c C s | j d�S )Nr� )r� )r$ r r r r� 3 s zValueProxy.getc C s | j d|f�S )Nr� )r� )r$ r� r r r r� 5 s zValueProxy.setN)r� r� )r* r, r- r� r� r� r� r� r r r r r 1 s r � BaseListProxy�__add__� __contains__� __delitem__� __getitem__�__len__�__mul__� __reversed__�__rmul__� __setitem__rC �count�extend�index�insertr� �remove�reverser� �__imul__c @ s e Zd Zdd� Zdd� ZdS )� ListProxyc C s | j d|f� | S )Nr+ )r� )r$ r� r r r �__iadd__A s zListProxy.__iadd__c C s | j d|f� | S )Nr0 )r� )r$ r� r r r r0 D s zListProxy.__imul__N)r* r, r- r2 r0 r r r r r1 @ s r1 � DictProxyr r� r� r� �has_key�popitem� setdefaultr� �Iterator� ArrayProxy� PoolProxy�apply� apply_asyncr} �imap�imap_unorderedr� �map� map_async�starmap� starmap_asyncr� Z AsyncResult)r; r? rA r< r= c @ s e Zd Zdd� Zdd� ZdS )r9 c C s | S )Nr )r$ r r r r� d s zPoolProxy.__enter__c C s | j � d S )N)r� )r$ r� r� r� r r r r� f s zPoolProxy.__exit__N)r* r, r- r� r� r r r r r9 c s c @ s e Zd ZdZdS )r a( Subclass of `BaseManager` which supports a number of shared object types. The types registered are those intended for the synchronization of threads, plus `dict`, `list` and `Namespace`. The `multiprocessing.Manager()` function creates started instances of this class. N)r* r, r- r. r r r r r m s �QueueZ JoinableQueuerd r^ �RLock� Semaphore�BoundedSemaphore� Condition�Barrier�Poolr �dictF)r� r� )r� )r r r )NNNT)T)r! r" r# r$ r% r& r' r( r) rC r* r+ r, r- r� r. r/ r� r0 )r" r# r$ r r% r) r� r� r� r4 r r r� r5 r6 r� r )r% r$ r) ) r: r; r} r< r= r� r> r? r@ rA r� )I�__all__rp r] r Zqueuer r� r � r �contextr r r r r r r r� Z view_typesr r Z view_typer r r9 r2 r{ r: rF rH rI r� ra rb Z XmlListenerZ XmlClientrX r r� r� r r� r� r� r� r r r r r r r r r r r1 r3 r� r8 Z BasePoolProxyr9 r rB rd r^ rC rD rE rF rG rH rI r r r r �<module> s� ; ] 4 __pycache__/queues.cpython-36.pyc 0000644 00000021363 15204243005 0012711 0 ustar 00 3 \ * � @ s� d ddgZ ddlZddlZddlZddlZddlZddlZddlZddlm Z m Z ddl Z ddl m Z ddl mZ ejjZdd lmZmZmZmZmZ G d d � d e�Ze� ZG d d� de�ZG d d� de�Z dS ) �Queue� SimpleQueue� JoinableQueue� N)�Empty�Full� )� connection)�context)�debug�info�Finalize�register_after_fork� is_exitingc @ s� e Zd Zd(dd�Zdd� Zdd� Zdd � Zd)d d �Zd*dd�Zdd� Z dd� Z dd� Z dd� Z dd� Z dd� Zd d � Zd d � Zd d!� Zed"d#� �Zed$d%� �Zed&d'� �Zd S )+r r c C s� |dkrddl m} || _tjdd�\| _| _|j� | _t j � | _ t j dkrTd | _n |j� | _|j|�| _d| _| j� t j dkr�t| tj� d S )Nr r )� SEM_VALUE_MAXF)�duplex�win32)Z synchronizer �_maxsizer �Pipe�_reader�_writer�Lock�_rlock�os�getpid�_opid�sys�platform�_wlockZBoundedSemaphore�_sem� _ignore_epipe� _after_forkr r )�self�maxsize�ctx� r$ �./usr/lib64/python3.6/multiprocessing/queues.py�__init__$ s zQueue.__init__c C s. t j| � | j| j| j| j| j| j| j| j fS )N) r �assert_spawningr r r r r r r r )r! r$ r$ r% � __getstate__9 s zQueue.__getstate__c C s0 |\| _ | _| _| _| _| _| _| _| j� d S )N) r r r r r r r r r )r! �stater$ r$ r% � __setstate__> s $zQueue.__setstate__c C sb t d� tjtj� �| _tj� | _d | _d | _ d| _ d| _ d | _ | j j| _| jj| _| jj| _d S )NzQueue._after_fork()F)r � threading� Conditionr � _notempty� collections�deque�_buffer�_thread� _jointhread�_joincancelled�_closed�_closer � send_bytes� _send_bytesr � recv_bytes� _recv_bytes�poll�_poll)r! r$ r$ r% r C s zQueue._after_forkTNc C s\ | j s t�| jj||�s t�| j�. | jd kr8| j� | jj |� | jj � W d Q R X d S )N) r4 �AssertionErrorr �acquirer r- r1 � _start_threadr0 �append�notify)r! �obj�block�timeoutr$ r$ r% �putP s z Queue.putc C s� |r2|d kr2| j � | j� }W d Q R X | jj� nr|rBtj� | }| j j||�sTt�zB|rv|tj� }| j|�s�t�n | j� s�t�| j� }| jj� W d | j j� X t j |�S )N) r r9 r �release�timeZ monotonicr= r r; �_ForkingPickler�loads)r! rB rC �resZdeadliner$ r$ r% �get[ s& z Queue.getc C s | j | jjj� S )N)r r �_semlockZ _get_value)r! r$ r$ r% �qsizes s z Queue.qsizec C s | j � S )N)r; )r! r$ r$ r% �emptyw s z Queue.emptyc C s | j jj� S )N)r rK �_is_zero)r! r$ r$ r% �fullz s z Queue.fullc C s | j d�S )NF)rJ )r! r$ r$ r% � get_nowait} s zQueue.get_nowaitc C s | j |d�S )NF)rD )r! rA r$ r$ r% � put_nowait� s zQueue.put_nowaitc C s2 d| _ z| jj� W d | j}|r,d | _|� X d S )NT)r4 r �closer5 )r! rR r$ r$ r% rR � s z Queue.closec C s$ t d� | jst�| jr | j� d S )NzQueue.join_thread())r r4 r< r2 )r! r$ r$ r% � join_thread� s zQueue.join_threadc C s6 t d� d| _y| jj� W n tk r0 Y nX d S )NzQueue.cancel_join_thread()T)r r3 r2 Zcancel�AttributeError)r! r$ r$ r% �cancel_join_thread� s zQueue.cancel_join_threadc C s� t d� | jj� tjtj| j| j| j| j | j j | j fdd�| _ d| j _t d� | j j� t d� | js�t| j tjtj| j �gd d�| _t| tj| j| jgd d�| _d S ) NzQueue._start_thread()ZQueueFeederThread)�target�args�nameTzdoing self._thread.start()z ... done self._thread.start()� )Z exitpriority� ���)r r0 �clearr+ ZThreadr �_feedr- r7 r r rR r r1 Zdaemon�startr3 r �_finalize_join�weakref�refr2 �_finalize_closer5 )r! r$ r$ r% r> � s( zQueue._start_threadc C s4 t d� | � }|d k r(|j� t d� nt d� d S )Nzjoining queue threadz... queue thread joinedz ... queue thread already dead)r �join)ZtwrZthreadr$ r$ r% r_ � s zQueue._finalize_joinc C s. t d� |� | jt� |j� W d Q R X d S )Nz telling queue thread to quit)r r? � _sentinelr@ )�buffer�notemptyr$ r$ r% rb � s zQueue._finalize_closec C sP t d� |j}|j}|j}| j} t} tjdkr<|j} |j} nd } �xy�|� z| sX|� W d |� X ybx\| � } | | kr�t d� |� d S tj | �} | d kr�|| � qh| � z || � W d | � X qhW W n t k r� Y nX W qD t k �rF } zJ|�rt |dd�t jk�rd S t� �r&td|� d S dd l}|j� W Y d d }~X qDX qDW d S )Nz$starting thread to feed data to piper z%feeder thread got sentinel -- exiting�errnor zerror in queue thread: %s)r r= rE �wait�popleftrd r r rG �dumps� IndexError� Exception�getattrrg ZEPIPEr r � traceback� print_exc)re rf r6 Z writelockrR Z ignore_epipeZnacquireZnreleaseZnwaitZbpopleft�sentinelZwacquireZwreleaserA �ern r$ r$ r% r] � sR z Queue._feed)r )TN)TN)�__name__� __module__� __qualname__r& r( r* r rD rJ rL rM rO rP rQ rR rS rU r> � staticmethodr_ rb r] r$ r$ r$ r% r " s$ c @ s@ e Zd Zddd�Zdd� Zdd� Zdd d �Zd d � Zdd� Zd S )r r c C s* t j| ||d� |jd�| _|j� | _d S )N)r# r )r r& Z Semaphore�_unfinished_tasksr, �_cond)r! r" r# r$ r$ r% r& s zJoinableQueue.__init__c C s t j| �| j| jf S )N)r r( rw rv )r! r$ r$ r% r( s zJoinableQueue.__getstate__c C s, t j| |d d� � |dd � \| _| _d S )N� ���ry )r r* rw rv )r! r) r$ r$ r% r* s zJoinableQueue.__setstate__TNc C sx | j s t�| jj||�s t�| j�J | j�8 | jd kr@| j� | j j |� | j j � | jj � W d Q R X W d Q R X d S )N)r4 r< r r= r r- rw r1 r> r0 r? rv rE r@ )r! rA rB rC r$ r$ r% rD s zJoinableQueue.putc C s@ | j �0 | jjd�s td��| jjj� r2| j j� W d Q R X d S )NFz!task_done() called too many times)rw rv r= � ValueErrorrK rN Z notify_all)r! r$ r$ r% � task_done' s zJoinableQueue.task_donec C s, | j � | jjj� s | j j� W d Q R X d S )N)rw rv rK rN rh )r! r$ r$ r% rc . s zJoinableQueue.join)r )TN) rr rs rt r& r( r* rD r{ rc r$ r$ r$ r% r s c @ s< e Zd Zdd� Zdd� Zdd� Zdd� Zd d � Zd d � Zd S )r c C sH t jdd�\| _| _|j� | _| jj| _tj dkr:d | _ n |j� | _ d S )NF)r r ) r r r r r r r: r; r r r )r! r# r$ r$ r% r& 9 s zSimpleQueue.__init__c C s | j � S )N)r; )r! r$ r$ r% rM B s zSimpleQueue.emptyc C s t j| � | j| j| j| jfS )N)r r' r r r r )r! r$ r$ r% r( E s zSimpleQueue.__getstate__c C s" |\| _ | _| _| _| j j| _d S )N)r r r r r: r; )r! r) r$ r$ r% r* I s zSimpleQueue.__setstate__c C s& | j � | jj� }W d Q R X tj|�S )N)r r r8 rG rH )r! rI r$ r$ r% rJ M s zSimpleQueue.getc C sD t j|�}| jd kr"| jj|� n | j� | jj|� W d Q R X d S )N)rG rj r r r6 )r! rA r$ r$ r% rD S s zSimpleQueue.putN) rr rs rt r& rM r( r* rJ rD r$ r$ r$ r% r 7 s ) �__all__r r r+ r. rF r` rg Zqueuer r Z_multiprocessing� r r Z reductionZForkingPicklerrG �utilr r r r r �objectr rd r r r$ r$ r$ r% �<module> s$ b * __pycache__/process.cpython-36.opt-2.pyc 0000644 00000014455 15204243005 0014024 0 ustar 00 3 \�# � @ s d ddgZ ddlZddlZddlZddlZddlmZ yejjej � �Z W n e k rb dZ Y nX dd� Z dd� Z dd � ZG d d � d e�ZG d d � d e�ZG d d� de�Ze� aejd�ae� a[i Zx<eejj � �D ]*\Z Z e dd� dko�de kr�e ee < q�W e� Z dS )� BaseProcess�current_process�active_children� N)�WeakSetc C s t S )N)�_current_process� r r �//usr/lib64/python3.6/multiprocessing/process.pyr # s c C s t � tt�S )N)�_cleanup�list� _childrenr r r r r ) s c C s. x(t t�D ] } | jj� d k r tj| � q W d S )N)r r �_popen�poll�discard)�pr r r r 4 s r c @ s� e Zd Zdd� Zdddf i fdd�dd�Zdd� Zd d � Zd d � Zd$d d�Zdd� Z e dd� �Z e j dd� �Z e dd� �Z e j dd� �Z e dd� �Zej dd� �Ze dd� �Ze d d � �ZeZe d d � �Zd d!� Zd"d#� ZdS )%r c C s t �d S )N)�NotImplementedError)�selfr r r �_PopenD s zBaseProcess._PopenN)�daemonc C s� t t�}tj|f | _tjj� | _tj� | _d | _ || _ t |�| _ t |�| _|prt| �jd djdd� | jD �� | _|d k r�|| _tj| � d S )N�-�:c s s | ]}t |�V qd S )N)�str)�.0�ir r r � <genexpr>S s z'BaseProcess.__init__.<locals>.<genexpr>)�next�_process_counterr � _identity�_config�copy�os�getpid� _parent_pidr �_target�tuple�_args�dict�_kwargs�type�__name__�join�_namer � _dangling�add)r �group�target�name�args�kwargsr �countr r r �__init__G s zBaseProcess.__init__c C s | j r| j | j| j� d S )N)r" r$ r& )r r r r �runX s zBaseProcess.runc C s6 t � | j| �| _| jj| _| `| `| `tj | � d S )N) r r r �sentinel� _sentinelr" r$ r& r r, )r r r r �start_ s zBaseProcess.startc C s | j j� d S )N)r � terminate)r r r r r8 p s zBaseProcess.terminatec C s" | j j|�}|d k r tj| � d S )N)r �waitr r )r Ztimeout�resr r r r) v s zBaseProcess.joinc C sB | t kr dS | jd krdS | jj� }|d kr0dS tj| � dS d S )NTF)r r r r r )r � returncoder r r �is_alive� s zBaseProcess.is_alivec C s | j S )N)r* )r r r r r/ � s zBaseProcess.namec C s || _ d S )N)r* )r r/ r r r r/ � s c C s | j jdd�S )Nr F)r �get)r r r r r � s zBaseProcess.daemonc C s || j d< d S )Nr )r )r Zdaemonicr r r r � s c C s | j d S )N�authkey)r )r r r r r> � s zBaseProcess.authkeyc C s t |�| jd< d S )Nr> )�AuthenticationStringr )r r> r r r r> � s c C s | j d kr| j S | j j� S )N)r r )r r r r �exitcode� s zBaseProcess.exitcodec C s" | t krtj� S | jo | jjS d S )N)r r r r �pid)r r r r �ident� s zBaseProcess.identc C s( y| j S tk r" td��Y nX d S )Nzprocess not started)r6 �AttributeError� ValueError)r r r r r5 � s zBaseProcess.sentinelc C s� | t krd}n>| jtj� kr"d}n*| jd kr2d}n| jj� d k rH| j}nd}t|�tkrv|dkrfd}ndt j ||� }dt| �j | j || j r�dp�d f S ) NZstarted�unknown�initialr Zstoppedz stopped[%s]z<%s(%s, %s%s)>z daemon� )r r! r r r r r@ r' �int�_exitcode_to_namer= r( r* r )r Zstatusr r r �__repr__� s zBaseProcess.__repr__c C sJ ddl m}m} �zy|| jd k r,|j| j� tjd�at� a |j � t }| a z|j j � |j� W d ~X |jd� z| j� d}W d |j� X W n� tk r� } zL|js�d}n:t|jd t�r�|jd }n tjjt|jd �d � d}W Y d d }~X n2 d}dd l}tjjd| j � |j � Y nX W d |jd| � |j � X |S )N� )�util�contextz child process calling self.run()r � z Process %s: z process exiting with exitcode %d) rG rL rM Z _start_methodZ_force_start_method� itertoolsr2 r �setr Z _close_stdinr Z_finalizer_registry�clearZ_run_after_forkers�infor4 Z_exit_function� SystemExitr0 � isinstancerH �sys�stderr�writer � tracebackr/ � print_excZ_flush_std_streams)r rL rM Z old_processr@ �erX r r r � _bootstrap� sB zBaseProcess._bootstrap)N)r( � __module__� __qualname__r r3 r4 r7 r8 r) r< �propertyr/ �setterr r> r@ rB rA r5 rJ r[ r r r r r > s( c @ s e Zd Zdd� ZdS )r? c C s, ddl m} |� d kr td��tt| �ffS )NrK )�get_spawning_popenzJPickling an AuthenticationString object is disallowed for security reasons)rM r` � TypeErrorr? �bytes)r r` r r r � __reduce__ s z AuthenticationString.__reduce__N)r( r\ r] rc r r r r r? s r? c @ s e Zd Zdd� ZdS )� _MainProcessc C s2 f | _ d| _d | _d | _ttjd��dd�| _d S )NZ MainProcess� z/mp)r> Z semprefix)r r* r! r r? r �urandomr )r r r r r3 - s z_MainProcess.__init__N)r( r\ r] r3 r r r r rd + s rd rK � ZSIG�_) �__all__r rU �signalrO � _weakrefsetr �path�abspath�getcwdZ ORIGINAL_DIR�OSErrorr r r �objectr rb r? rd r r2 r rP r rI r �__dict__�itemsr/ Zsignumr+ r r r r �<module> s2 ` __pycache__/util.cpython-36.opt-1.pyc 0000644 00000022730 15204243005 0013315 0 ustar 00 3 \n. � @ s� d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ ddlm Z ddddd d d d d ddddddgZ d Z dZ dZ dZdZdZdZdadadd� Zd d� Zd d� Zd d� Zd d � Zd6d d �Zd!d � Zej� Z ej � Z d"d#� Z d$d � Z i Z!ej � Z"G d%d� de#�Z$d7d&d'�Z%d(d � Z&da'eee%e j(e j)fd)d*�Z*ej+e*� G d+d� de#�Z,G d,d� dej-�Z.ye j/d-�Z0W n e1k �r� d.Z0Y nX d/d� Z2d0d1� Z3d2d3� Z4d4d5� Z5dS )8� N)� _args_from_interpreter_flags� )�process� sub_debug�debug�info� sub_warning� get_logger� log_to_stderr� get_temp_dir�register_after_fork� is_exiting�Finalize�ForkAwareThreadLock�ForkAwareLocal�close_all_fds_except�SUBDEBUG� SUBWARNING� � � � Zmultiprocessingz+[%(levelname)s/%(processName)s] %(message)sFc G s t rt jt| f|�� d S )N)�_logger�logr )�msg�args� r �,/usr/lib64/python3.6/multiprocessing/util.pyr , s c G s t rt jt| f|�� d S )N)r r �DEBUG)r r r r r r 0 s c G s t rt jt| f|�� d S )N)r r �INFO)r r r r r r 4 s c G s t rt jt| f|�� d S )N)r r r )r r r r r r 8 s c C s| ddl } | j� z\tsj| jt�adt_ttd�rFtjt � tj t � n$tj j t f i f� tj j t f i f� W d| j� X tS )z0 Returns logger used by multiprocessing r N� unregister)�loggingZ _acquireLockr Z getLogger� LOGGER_NAMEZ propagate�hasattr�atexitr �_exit_function�registerZ _exithandlers�remove�appendZ _releaseLock)r! r r r r < s c C sJ ddl }t� }|jt�}|j� }|j|� |j|� | rB|j| � dat S )zB Turn on logging and add a handler which prints to stderr r NT) r! r Z Formatter�DEFAULT_LOGGING_FORMATZ StreamHandlerZ setFormatterZ addHandlerZsetLevel�_log_to_stderrr )�levelr! ZloggerZ formatterZhandlerr r r r W s c C sd t j� jjd�} | d kr`dd l}dd l}|jdd�} td| � td |j | gdd� | t j� jd< | S ) N�tempdirr zpymp-)�prefixzcreated temp directory %s�d )r � exitpriorityi����) r �current_processZ_config�get�shutil�tempfileZmkdtempr r Zrmtree)r, r2 r3 r r r r m s c C sj t tj� �} | j� xP| D ]H\\}}}}y ||� W q tk r` } ztd|� W Y d d }~X qX qW d S )Nz after forker raised exception %s)�list�_afterfork_registry�items�sort� Exceptionr )r6 �indexZident�func�obj�er r r �_run_after_forkers s r= c C s | t tt�t| �|f< d S )N)r5 �next�_afterfork_counter�id)r; r: r r r r � s c @ sL e Zd ZdZf ddfdd�Zdeeejfdd�Z dd� Z d d � Z d d � Z dS ) r zA Class which supports object finalization using weakrefs Nc C sT |d k rt j|| �| _n || _|| _|p*i | _|tt�f| _t j � | _ | t | j< d S )N) �weakref�ref�_weakref� _callback�_args�_kwargsr> �_finalizer_counter�_key�os�getpid�_pid�_finalizer_registry)�selfr; �callbackr �kwargsr/ r r r �__init__� s zFinalize.__init__c C s� y || j = W n tk r( |d� Y nbX | j|� krD|d� d}n$|d| j| j| j� | j| j| j�}d | _ | _ | _ | _| _ |S dS )zQ Run the callback unless it has already been called or cancelled z finalizer no longer registeredz+finalizer ignored because different processNz/finalizer calling %s with args %s and kwargs %s)rH �KeyErrorrK rD rE rF rC )rM �wrrL r rJ �resr r r �__call__� s zFinalize.__call__c C sD y t | j= W n tk r Y n X d | _ | _ | _ | _| _dS )z3 Cancel finalization of the object N)rL rH rQ rC rD rE rF )rM r r r �cancel� s zFinalize.cancelc C s | j tkS )zS Return whether this finalizer is still waiting to invoke callback )rH rL )rM r r r � still_active� s zFinalize.still_activec C s� y | j � }W n ttfk r( d }Y nX |d kr>d| jj S d| jjt| jd| j�f }| jrr|dt| j� 7 }| j r�|dt| j � 7 }| j d d k r�|dt| j d � 7 }|d S ) Nz<%s object, dead>z<%s object, callback=%s�__name__z, args=z , kwargs=r z, exitprority=�>) rC �AttributeError� TypeError� __class__rW �getattrrD rE �strrF rH )rM r; �xr r r �__repr__� s zFinalize.__repr__) rW � __module__� __qualname__�__doc__rP rL r rI rJ rT rU rV r_ r r r r r � s c s� t dkr dS �dkr dd� � n �fdd�� � fdd�tt �D �}|jdd� xX|D ]P}t j|�}|dk rRtd |� y |� W qR tk r� d dl}|j� Y qRX qRW �dkr�t j� dS ) z� Run all finalizers whose exit priority is not None and at least minpriority Finalizers with highest priority are called first; finalizers with the same priority will be called in reverse order of creation. Nc S s | d d k S )Nr r )�pr r r �<lambda>� s z!_run_finalizers.<locals>.<lambda>c s | d d k o| d � kS )Nr r )rc )� minpriorityr r rd � s c s g | ]}� |�r|�qS r r )�.0�key)�fr r � <listcomp>� s z#_run_finalizers.<locals>.<listcomp>T)�reversez calling %sr ) rL r4 r7 r1 r r8 � traceback� print_exc�clear)re �keysrg � finalizerrk r )rh re r �_run_finalizers� s$ rp c C s t p t dkS )z6 Returns true if the process is shutting down N)�_exitingr r r r r s c C s� t s�da | d� |d� |d� |� d k rzx*|� D ] }|jr2| d|j� |jj� q2W x"|� D ]}| d|j� |j� q^W |d� |� d S )NTzprocess shutting downz2running all "atexit" finalizers with priority >= 0r z!calling terminate() for daemon %sz calling join() for process %sz)running the remaining "atexit" finalizers)rq Zdaemon�nameZ_popenZ terminate�join)r r rp �active_childrenr0 rc r r r r% s r% c @ s, e Zd Zdd� Zdd� Zdd� Zdd� Zd S ) r c C s | j � t| tj � d S )N)�_resetr r )rM r r r rP K s z ForkAwareThreadLock.__init__c C s" t j� | _| jj| _| jj| _d S )N)� threadingZLock�_lock�acquire�release)rM r r r ru O s zForkAwareThreadLock._resetc C s | j j� S )N)rw � __enter__)rM r r r rz T s z ForkAwareThreadLock.__enter__c G s | j j|� S )N)rw �__exit__)rM r r r r r{ W s z ForkAwareThreadLock.__exit__N)rW r` ra rP ru rz r{ r r r r r J s c @ s e Zd Zdd� Zdd� ZdS )r c C s t | dd� � d S )Nc S s | j j� S )N)�__dict__rm )r; r r r rd ] s z)ForkAwareLocal.__init__.<locals>.<lambda>)r )rM r r r rP \ s zForkAwareLocal.__init__c C s t | �f fS )N)�type)rM r r r � __reduce__^ s zForkAwareLocal.__reduce__N)rW r` ra rP r~ r r r r r [ s � SC_OPEN_MAX� c C sR t | �dtg } | j� x4tt| �d �D ] }tj| | d | |d � q*W d S )Nr ���)r4 �MAXFDr7 �range�lenrI � closerange)Zfds�ir r r r j s c C s� t jd krd S yt jj� W n ttfk r4 Y nX y@tjtjtj�} yt| dd�t _W n tj| � � Y nX W n ttfk r� Y nX d S )NF)�closefd) �sys�stdin�close�OSError� ValueErrorrI �open�devnull�O_RDONLY)�fdr r r � _close_stdint s r� c C sT yt jj� W n ttfk r& Y nX yt jj� W n ttfk rN Y nX d S )N)r� �stdout�flushrY r� �stderrr r r r �_flush_std_streams� s r� c C st dd l }tttt|���}tj� \}}z2|j|tj| �gd|d d ddddd d ||ddd �S tj |� tj |� X d S ) Nr Tr Fr� r� r� r� r� r� ) �_posixsubprocess�tuple�sorted�map�intrI �pipeZ fork_exec�fsencoder� )�pathr Zpassfdsr� Z errpipe_readZ errpipe_writer r r �spawnv_passfds� s r� )N)N)6rI � itertoolsr� rA r$ rv � subprocessr � r �__all__ZNOTSETr r r r r" r) r r* r r r r r r r �WeakValueDictionaryr5 �countr? r= r rL rG �objectr rp r rq rt r0 r% r& r Zlocalr �sysconfr� r8 r r� r� r� r r r r �<module> sf S ,( __pycache__/context.cpython-36.opt-2.pyc 0000644 00000026523 15204243005 0014031 0 ustar 00 3 \�* � @ s� d dl Z d dlZd dlZddlmZ ddlmZ g ZG dd� de�ZG dd� de�Z G d d � d e�Z G d d � d e�Z G d d� de �Z G dd� dej�ZG dd� de �Zedd� ee�D ��e_ejdk�rPG dd� dej�ZG dd� dej�ZG dd� dej�ZG d d � d e �ZG d d � d e �ZG d d!� d!e �Ze� e� e� d"�Zeed# �Zn8G d$d� dej�ZG d%d � d e �Zd&e� iZeed& �Zd'd(� Z ej � Z d)d*� Z d+d,� Z d-d.� Z!dS )/� N� )�process)� reductionc @ s e Zd ZdS )� ProcessErrorN)�__name__� __module__� __qualname__� r r �//usr/lib64/python3.6/multiprocessing/context.pyr s r c @ s e Zd ZdS )�BufferTooShortN)r r r r r r r r s r c @ s e Zd ZdS )� TimeoutErrorN)r r r r r r r r s r c @ s e Zd ZdS )�AuthenticationErrorN)r r r r r r r r s r c @ sV e Zd ZeZeZeZeZeej �Z eej �Z dd� Z dd� Z dBdd�Z dd � Zd d � ZdCd d�ZdDdd�ZdEdd�Zdd� ZdFdd�ZdGdd�ZdHdd �Zd d � Zd d f d fd d �Zd!d"� Zd#d$� Zdd%�d&d'�Zdd%�d(d)�Z d*d+� Z d,d-� Z dId.d/�Z d0d1� Z d2d3� Z!d4d5� Z"dJd6d7�Z#dKd9d:�Z$dLd;d<�Z%e&d=d>� �Z'e'j(d?d>� �Z'd@dA� Z)d S )M� BaseContextc C s" t j� }|d krtd��n|S d S )Nz cannot determine number of cpus)�os� cpu_count�NotImplementedError)�selfZnumr r r r ( s zBaseContext.cpu_countc C s& ddl m} || j� d�}|j� |S )Nr )� SyncManager)�ctx)Zmanagersr � get_context�start)r r �mr r r �Manager0 s zBaseContext.ManagerTc C s ddl m} ||�S )Nr )�Pipe)� connectionr )r Zduplexr r r r r ; s zBaseContext.Pipec C s ddl m} || j� d�S )Nr )�Lock)r )� synchronizer r )r r r r r r @ s zBaseContext.Lockc C s ddl m} || j� d�S )Nr )�RLock)r )r r r )r r r r r r E s zBaseContext.RLockNc C s ddl m} ||| j� d�S )Nr )� Condition)r )r r r )r �lockr r r r r J s zBaseContext.Conditionr c C s ddl m} ||| j� d�S )Nr )� Semaphore)r )r r r )r �valuer r r r r O s zBaseContext.Semaphorec C s ddl m} ||| j� d�S )Nr )�BoundedSemaphore)r )r r" r )r r! r" r r r r" T s z BaseContext.BoundedSemaphorec C s ddl m} || j� d�S )Nr )�Event)r )r r# r )r r# r r r r# Y s zBaseContext.Eventc C s ddl m} ||||| j� d�S )Nr )�Barrier)r )r r$ r )r Zparties�actionZtimeoutr$ r r r r$ ^ s zBaseContext.Barrierr c C s ddl m} ||| j� d�S )Nr )�Queue)r )�queuesr& r )r �maxsizer& r r r r& c s zBaseContext.Queuec C s ddl m} ||| j� d�S )Nr )� JoinableQueue)r )r' r) r )r r( r) r r r r) h s zBaseContext.JoinableQueuec C s ddl m} || j� d�S )Nr )� SimpleQueue)r )r' r* r )r r* r r r r* m s zBaseContext.SimpleQueuec C s" ddl m} |||||| j� d�S )Nr )�Pool)�context)Zpoolr+ r )r Z processesZ initializerZinitargsZmaxtasksperchildr+ r r r r+ r s zBaseContext.Poolc G s ddl m} ||f|�� S )Nr )�RawValue)� sharedctypesr- )r �typecode_or_type�argsr- r r r r- y s zBaseContext.RawValuec C s ddl m} |||�S )Nr )�RawArray)r. r1 )r r/ �size_or_initializerr1 r r r r1 ~ s zBaseContext.RawArray)r c G s& ddl m} ||f|�|| j� d��S )Nr )�Value)r r )r. r3 r )r r/ r r0 r3 r r r r3 � s zBaseContext.Valuec C s ddl m} ||||| j� d�S )Nr )�Array)r r )r. r4 r )r r/ r2 r r4 r r r r4 � s zBaseContext.Arrayc C s, t jdkr(tt dd�r(ddlm} |� d S )N�win32�frozenFr )�freeze_support)�sys�platform�getattr�spawnr7 )r r7 r r r r7 � s zBaseContext.freeze_supportc C s ddl m} |� S )Nr )� get_logger)�utilr< )r r< r r r r< � s zBaseContext.get_loggerc C s ddl m} ||�S )Nr )� log_to_stderr)r= r> )r �levelr> r r r r> � s zBaseContext.log_to_stderrc C s ddl m} d S )Nr )r )� r )r r r r r �allow_connection_pickling� s z%BaseContext.allow_connection_picklingc C s ddl m} ||� d S )Nr )�set_executable)r; rB )r � executablerB r r r rB � s zBaseContext.set_executablec C s ddl m} ||� d S )Nr )�set_forkserver_preload)� forkserverrD )r Z module_namesrD r r r rD � s z"BaseContext.set_forkserver_preloadc C sF |d kr | S y t | }W n tk r8 td| ��Y nX |j� |S )Nzcannot find context for %r)�_concrete_contexts�KeyError� ValueError�_check_available)r �methodr r r r r � s zBaseContext.get_contextFc C s | j S )N)�_name)r � allow_noner r r �get_start_method� s z BaseContext.get_start_methodc C s t d��d S )Nz+cannot set start method of concrete context)rH )r rJ �forcer r r �set_start_method� s z BaseContext.set_start_methodc C s t � jd�S )Nr )�globals�get)r r r r �reducer� s zBaseContext.reducerc C s |t � d< d S )Nr )rP )r r r r r rR � s c C s d S )Nr )r r r r rI � s z BaseContext._check_available)T)N)r )r )NN)r )r )N)N)F)F)*r r r r r r r � staticmethodr Zcurrent_processZactive_childrenr r r r r r r r" r# r$ r& r) r* r+ r- r1 r3 r4 r7 r< r> rA rB rD r rM rO �propertyrR �setterrI r r r r r sJ r c @ s e Zd ZdZedd� �ZdS )�ProcessNc C s t j� jj| �S )N)�_default_contextr rV �_Popen)� process_objr r r rX � s zProcess._Popen)r r r � _start_methodrS rX r r r r rV � s rV c sF e Zd ZeZdd� Zd � fdd� Zddd�Zdd d �Zd d � Z� Z S )�DefaultContextc C s || _ d | _d S )N)rW �_actual_context)r r, r r r �__init__� s zDefaultContext.__init__Nc s0 |d kr | j d kr| j| _ | j S t� j|�S d S )N)r\ rW �superr )r rJ )� __class__r r r � s zDefaultContext.get_contextFc C s> | j d k r| rtd��|d kr.|r.d | _ d S | j|�| _ d S )Nz context has already been set)r\ � RuntimeErrorr )r rJ rN r r r rO � s z DefaultContext.set_start_methodc C s" | j d kr|rd S | j| _ | j jS )N)r\ rW rK )r rL r r r rM � s z DefaultContext.get_start_methodc C s, t jdkrdgS tjr dddgS ddgS d S )Nr5 r; �forkrE )r8 r9 r �HAVE_SEND_HANDLE)r r r r �get_all_start_methods� s z$DefaultContext.get_all_start_methods)N)F)F) r r r rV r] r rO rM rc � __classcell__r r )r_ r r[ � s r[ c c s | ]}|d dkr|V qdS )r �_Nr )�.0�xr r r � <genexpr> s rh r5 c @ s e Zd ZdZedd� �ZdS )� ForkProcessra c C s ddl m} || �S )Nr )�Popen)Z popen_forkrj )rY rj r r r rX s zForkProcess._PopenN)r r r rZ rS rX r r r r ri s ri c @ s e Zd ZdZedd� �ZdS )� SpawnProcessr; c C s ddl m} || �S )Nr )rj )Zpopen_spawn_posixrj )rY rj r r r rX s zSpawnProcess._PopenN)r r r rZ rS rX r r r r rk s rk c @ s e Zd ZdZedd� �ZdS )�ForkServerProcessrE c C s ddl m} || �S )Nr )rj )Zpopen_forkserverrj )rY rj r r r rX s zForkServerProcess._PopenN)r r r rZ rS rX r r r r rl s rl c @ s e Zd ZdZeZdS )� ForkContextra N)r r r rK ri rV r r r r rm % s rm c @ s e Zd ZdZeZdS )� SpawnContextr; N)r r r rK rk rV r r r r rn ) s rn c @ s e Zd ZdZeZdd� ZdS )�ForkServerContextrE c C s t jstd��d S )Nz%forkserver start method not available)r rb rH )r r r r rI 0 s z"ForkServerContext._check_availableN)r r r rK rl rV rI r r r r ro - s ro )ra r; rE ra c @ s e Zd ZdZedd� �ZdS )rk r; c C s ddl m} || �S )Nr )rj )Zpopen_spawn_win32rj )rY rj r r r rX ? s zSpawnProcess._PopenN)r r r rZ rS rX r r r r rk = s c @ s e Zd ZdZeZdS )rn r; N)r r r rK rk rV r r r r rn D s r; c C s t | t_d S )N)rF rW r\ )rJ r r r �_force_start_methodQ s rp c C s t tdd �S )N�spawning_popen)r: �_tlsr r r r �get_spawning_popenZ s rs c C s | t _d S )N)rr rq )�popenr r r �set_spawning_popen] s ru c C s t � d kr tdt| �j ��d S )NzF%s objects should only be shared between processes through inheritance)rs r` �typer )�objr r r �assert_spawning` s rx )"r r8 Z threadingr@ r r �__all__� Exceptionr r r r �objectr Z BaseProcessrV r[ �list�dirr9 ri rk rl rm rn ro rF rW rp Zlocalrr rs ru rx r r r r �<module> sD >' __pycache__/context.cpython-36.pyc 0000644 00000031452 15204243005 0013066 0 ustar 00 3 \�* � @ s� d dl Z d dlZd dlZddlmZ ddlmZ g ZG dd� de�ZG dd� de�Z G d d � d e�Z G d d � d e�Z G d d� de �Z G dd� dej�ZG dd� de �Zedd� ee�D ��e_ejdk�rPG dd� dej�ZG dd� dej�ZG dd� dej�ZG d d � d e �ZG d d � d e �ZG d d!� d!e �Ze� e� e� d"�Zeed# �Zn8G d$d� dej�ZG d%d � d e �Zd&e� iZeed& �Zd'd(� Z ej � Z d)d*� Z d+d,� Z d-d.� Z!dS )/� N� )�process)� reductionc @ s e Zd ZdS )� ProcessErrorN)�__name__� __module__� __qualname__� r r �//usr/lib64/python3.6/multiprocessing/context.pyr s r c @ s e Zd ZdS )�BufferTooShortN)r r r r r r r r s r c @ s e Zd ZdS )� TimeoutErrorN)r r r r r r r r s r c @ s e Zd ZdS )�AuthenticationErrorN)r r r r r r r r s r c @ sV e Zd ZeZeZeZeZeej �Z eej �Z dd� Z dd� Z dBdd�Z dd � Zd d � ZdCd d�ZdDdd�ZdEdd�Zdd� ZdFdd�ZdGdd�ZdHdd �Zd d � Zd d f d fd d �Zd!d"� Zd#d$� Zdd%�d&d'�Zdd%�d(d)�Z d*d+� Z d,d-� Z dId.d/�Z d0d1� Z d2d3� Z!d4d5� Z"dJd6d7�Z#dKd9d:�Z$dLd;d<�Z%e&d=d>� �Z'e'j(d?d>� �Z'd@dA� Z)d S )M� BaseContextc C s" t j� }|dkrtd��n|S dS )z(Returns the number of CPUs in the systemNz cannot determine number of cpus)�os� cpu_count�NotImplementedError)�selfZnumr r r r ( s zBaseContext.cpu_countc C s& ddl m} || j� d�}|j� |S )z�Returns a manager associated with a running server process The managers methods such as `Lock()`, `Condition()` and `Queue()` can be used to create shared objects. r )� SyncManager)�ctx)Zmanagersr � get_context�start)r r �mr r r �Manager0 s zBaseContext.ManagerTc C s ddl m} ||�S )z1Returns two connection object connected by a piper )�Pipe)� connectionr )r Zduplexr r r r r ; s zBaseContext.Pipec C s ddl m} || j� d�S )z#Returns a non-recursive lock objectr )�Lock)r )� synchronizer r )r r r r r r @ s zBaseContext.Lockc C s ddl m} || j� d�S )z Returns a recursive lock objectr )�RLock)r )r r r )r r r r r r E s zBaseContext.RLockNc C s ddl m} ||| j� d�S )zReturns a condition objectr )� Condition)r )r r r )r �lockr r r r r J s zBaseContext.Conditionr c C s ddl m} ||| j� d�S )zReturns a semaphore objectr )� Semaphore)r )r r r )r �valuer r r r r O s zBaseContext.Semaphorec C s ddl m} ||| j� d�S )z"Returns a bounded semaphore objectr )�BoundedSemaphore)r )r r" r )r r! r" r r r r" T s z BaseContext.BoundedSemaphorec C s ddl m} || j� d�S )zReturns an event objectr )�Event)r )r r# r )r r# r r r r# Y s zBaseContext.Eventc C s ddl m} ||||| j� d�S )zReturns a barrier objectr )�Barrier)r )r r$ r )r Zparties�actionZtimeoutr$ r r r r$ ^ s zBaseContext.Barrierr c C s ddl m} ||| j� d�S )zReturns a queue objectr )�Queue)r )�queuesr& r )r �maxsizer& r r r r& c s zBaseContext.Queuec C s ddl m} ||| j� d�S )zReturns a queue objectr )� JoinableQueue)r )r' r) r )r r( r) r r r r) h s zBaseContext.JoinableQueuec C s ddl m} || j� d�S )zReturns a queue objectr )� SimpleQueue)r )r' r* r )r r* r r r r* m s zBaseContext.SimpleQueuec C s" ddl m} |||||| j� d�S )z Returns a process pool objectr )�Pool)�context)Zpoolr+ r )r Z processesZ initializerZinitargsZmaxtasksperchildr+ r r r r+ r s zBaseContext.Poolc G s ddl m} ||f|�� S )zReturns a shared objectr )�RawValue)� sharedctypesr- )r �typecode_or_type�argsr- r r r r- y s zBaseContext.RawValuec C s ddl m} |||�S )zReturns a shared arrayr )�RawArray)r. r1 )r r/ �size_or_initializerr1 r r r r1 ~ s zBaseContext.RawArray)r c G s& ddl m} ||f|�|| j� d��S )z$Returns a synchronized shared objectr )�Value)r r )r. r3 r )r r/ r r0 r3 r r r r3 � s zBaseContext.Valuec C s ddl m} ||||| j� d�S )z#Returns a synchronized shared arrayr )�Array)r r )r. r4 r )r r/ r2 r r4 r r r r4 � s zBaseContext.Arrayc C s, t jdkr(tt dd�r(ddlm} |� dS )z�Check whether this is a fake forked process in a frozen executable. If so then run code specified by commandline and exit. �win32�frozenFr )�freeze_supportN)�sys�platform�getattr�spawnr7 )r r7 r r r r7 � s zBaseContext.freeze_supportc C s ddl m} |� S )zZReturn package logger -- if it does not already exist then it is created. r )� get_logger)�utilr< )r r< r r r r< � s zBaseContext.get_loggerc C s ddl m} ||�S )z8Turn on logging and add a handler which prints to stderrr )� log_to_stderr)r= r> )r �levelr> r r r r> � s zBaseContext.log_to_stderrc C s ddl m} dS )zVInstall support for sending connections and sockets between processes r )r N)� r )r r r r r �allow_connection_pickling� s z%BaseContext.allow_connection_picklingc C s ddl m} ||� dS )z�Sets the path to a python.exe or pythonw.exe binary used to run child processes instead of sys.executable when using the 'spawn' start method. Useful for people embedding Python. r )�set_executableN)r; rB )r � executablerB r r r rB � s zBaseContext.set_executablec C s ddl m} ||� dS )zkSet list of module names to try to load in forkserver process. This is really just a hint. r )�set_forkserver_preloadN)� forkserverrD )r Z module_namesrD r r r rD � s z"BaseContext.set_forkserver_preloadc C sF |d kr | S y t | }W n tk r8 td| ��Y nX |j� |S )Nzcannot find context for %r)�_concrete_contexts�KeyError� ValueError�_check_available)r �methodr r r r r � s zBaseContext.get_contextFc C s | j S )N)�_name)r � allow_noner r r �get_start_method� s z BaseContext.get_start_methodc C s t d��d S )Nz+cannot set start method of concrete context)rH )r rJ �forcer r r �set_start_method� s z BaseContext.set_start_methodc C s t � jd�S )z_Controls how objects will be reduced to a form that can be shared with other processes.r )�globals�get)r r r r �reducer� s zBaseContext.reducerc C s |t � d< d S )Nr )rP )r r r r r rR � s c C s d S )Nr )r r r r rI � s z BaseContext._check_available)T)N)r )r )NN)r )r )N)N)F)F)*r r r r r r r � staticmethodr Zcurrent_processZactive_childrenr r r r r r r r" r# r$ r& r) r* r+ r- r1 r3 r4 r7 r< r> rA rB rD r rM rO �propertyrR �setterrI r r r r r sJ r c @ s e Zd ZdZedd� �ZdS )�ProcessNc C s t j� jj| �S )N)�_default_contextr rV �_Popen)� process_objr r r rX � s zProcess._Popen)r r r � _start_methodrS rX r r r r rV � s rV c sF e Zd ZeZdd� Zd � fdd� Zddd�Zdd d �Zd d � Z� Z S )�DefaultContextc C s || _ d | _d S )N)rW �_actual_context)r r, r r r �__init__� s zDefaultContext.__init__Nc s0 |d kr | j d kr| j| _ | j S t� j|�S d S )N)r\ rW �superr )r rJ )� __class__r r r � s zDefaultContext.get_contextFc C s> | j d k r| rtd��|d kr.|r.d | _ d S | j|�| _ d S )Nz context has already been set)r\ � RuntimeErrorr )r rJ rN r r r rO � s z DefaultContext.set_start_methodc C s" | j d kr|rd S | j| _ | j jS )N)r\ rW rK )r rL r r r rM � s z DefaultContext.get_start_methodc C s, t jdkrdgS tjr dddgS ddgS d S )Nr5 r; �forkrE )r8 r9 r �HAVE_SEND_HANDLE)r r r r �get_all_start_methods� s z$DefaultContext.get_all_start_methods)N)F)F) r r r rV r] r rO rM rc � __classcell__r r )r_ r r[ � s r[ c c s | ]}|d dkr|V qdS )r �_Nr )�.0�xr r r � <genexpr> s rh r5 c @ s e Zd ZdZedd� �ZdS )� ForkProcessra c C s ddl m} || �S )Nr )�Popen)Z popen_forkrj )rY rj r r r rX s zForkProcess._PopenN)r r r rZ rS rX r r r r ri s ri c @ s e Zd ZdZedd� �ZdS )� SpawnProcessr; c C s ddl m} || �S )Nr )rj )Zpopen_spawn_posixrj )rY rj r r r rX s zSpawnProcess._PopenN)r r r rZ rS rX r r r r rk s rk c @ s e Zd ZdZedd� �ZdS )�ForkServerProcessrE c C s ddl m} || �S )Nr )rj )Zpopen_forkserverrj )rY rj r r r rX s zForkServerProcess._PopenN)r r r rZ rS rX r r r r rl s rl c @ s e Zd ZdZeZdS )� ForkContextra N)r r r rK ri rV r r r r rm % s rm c @ s e Zd ZdZeZdS )� SpawnContextr; N)r r r rK rk rV r r r r rn ) s rn c @ s e Zd ZdZeZdd� ZdS )�ForkServerContextrE c C s t jstd��d S )Nz%forkserver start method not available)r rb rH )r r r r rI 0 s z"ForkServerContext._check_availableN)r r r rK rl rV rI r r r r ro - s ro )ra r; rE ra c @ s e Zd ZdZedd� �ZdS )rk r; c C s ddl m} || �S )Nr )rj )Zpopen_spawn_win32rj )rY rj r r r rX ? s zSpawnProcess._PopenN)r r r rZ rS rX r r r r rk = s c @ s e Zd ZdZeZdS )rn r; N)r r r rK rk rV r r r r rn D s r; c C s t | t_d S )N)rF rW r\ )rJ r r r �_force_start_methodQ s rp c C s t tdd �S )N�spawning_popen)r: �_tlsr r r r �get_spawning_popenZ s rs c C s | t _d S )N)rr rq )�popenr r r �set_spawning_popen] s ru c C s t � d kr tdt| �j ��d S )NzF%s objects should only be shared between processes through inheritance)rs r` �typer )�objr r r �assert_spawning` s rx )"r r8 Z threadingr@ r r �__all__� Exceptionr r r r �objectr Z BaseProcessrV r[ �list�dirr9 ri rk rl rm rn ro rF rW rp Zlocalrr rs ru rx r r r r �<module> sD >' __pycache__/spawn.cpython-36.pyc 0000644 00000014453 15204243006 0012535 0 ustar 00 3 \�" � @ s. d dl Z d dlZd dlZd dlZddlmZmZ ddlmZ ddlm Z ddlm Z ddd d d d d gZ ej dkrzdZ dZn&ej dko�eedd�Z ejj� jd�Zer�e jjejd�anejadd � Zdd � Zdd� Zdd� Zdd � Zd&dd�Z dd� Z d d � Z d d � Z g Z d d � Z!d!d"� Z"d#d$� Z#d%d � Z$dS )'� N� )�get_start_method�set_start_method)�process)� reduction)�util�_main�freeze_support�set_executable�get_executable�get_preparation_data�get_command_line�import_main_path�win32F�frozenzpythonservice.exez python.exec C s | a d S )N)� _python_exe)Zexe� r �-/usr/lib64/python3.6/multiprocessing/spawn.pyr ) s c C s t S )N)r r r r r r - s c C s$ t | �dkr | d dkr dS dS dS )z= Return whether commandline indicates we are forking � r z--multiprocessing-forkTFN)�len)�argvr r r � is_forking4 s r c C sh t tj�rdi } xBtjdd� D ]0}|jd�\}}|dkrBd| |< q t|�| |< q W tf | � tj� dS )zE Run code for process object if this in not the main process r N�=�None)r �sysr �split�int� spawn_main�exit)�kwds�arg�name�valuer r r r > s c K sh t tdd�r(tjdgdd� | j� D � S d}|djdd � | j� D ��; }tj� }tg| d |dg S d S ) zJ Returns prefix of command line used for spawning a child process r Fz--multiprocessing-forkc S s g | ] }d | �qS )z%s=%rr )�.0�itemr r r � <listcomp>T s z$get_command_line.<locals>.<listcomp>z<from multiprocessing.spawn import spawn_main; spawn_main(%s)z, c s s | ]}d | V qdS )z%s=%rNr )r# r$ r r r � <genexpr>W s z#get_command_line.<locals>.<genexpr>z-cN)�getattrr � executable�items�joinr Z _args_from_interpreter_flagsr )r �progZoptsr r r r N s c C sj t tj�st�tjdkr<ddl}tj|| �}|j|t j �}nddl m } ||j _| }t|�}tj|� dS )z7 Run code specified by data received over pipe r r Nr )�semaphore_tracker)r r r �AssertionError�platform�msvcrtr Z steal_handleZopen_osfhandle�os�O_RDONLY� r, Z_semaphore_trackerZ_fdr r )Z pipe_handleZ parent_pidZ tracker_fdr/ Z new_handle�fdr, Zexitcoder r r r \ s r c C s^ t j| ddd��@}dtj� _z$tjj|�}t|� tjj|�}W d tj� `X W d Q R X |j � S )N�rbT)�closefd) r0 �fdopenr �current_process� _inheritingr �pickle�load�prepare� _bootstrap)r3 Z from_parentZpreparation_data�selfr r r r m s c C s t tj� dd�rtd��d S )Nr8 Fa An attempt has been made to start a new process before the current process has finished its bootstrapping phase. This probably means that you are not using fork to start your child processes and you have forgotten to use the proper idiom in the main module: if __name__ == '__main__': freeze_support() ... The "freeze_support()" line can be omitted if the program is not going to be frozen to produce an executable.)r' r r7 � RuntimeErrorr r r r �_check_not_importing_mainy s r? c C s t � ttjtj� jd�}tjdk r2tjj� |d< t j j � }y|j d�}W n t k r^ Y n X tj||< |j| |t jtjtj� t� d� t jd }t|jdd�}|dk r�||d< nht jd ks�t o�t �r t|d d�}|dk �r tj j|� o�tjdk �r tj jtj|�}tj j |�|d < |S ) zM Return info about parent needed by child to unpickle process object )� log_to_stderr�authkeyN� log_levelr2 )r! �sys_path�sys_argv�orig_dir�dir� start_method�__main__r! �init_main_from_namer �__file__�init_main_from_path) r? �dictr Z_log_to_stderrr r7 rA Z_loggerZgetEffectiveLevelr �path�copy�index� ValueError� ORIGINAL_DIR�updater r0 �getcwdr �modulesr' �__spec__r. �WINEXE� WINSERVICE�isabsr* �normpath)r! �drC �i� main_moduleZ main_mod_name� main_pathr r r r � s> c C s� d| kr| d t j� _d| kr,| d t j� _d| krD| d rDtj� d| kr^tj� j| d � d| krp| d t_ d| kr�| d t_ d| kr�t j | d � d| kr�| d t _ d | kr�t| d d d � d | kr�t| d � nd | kr�t| d � dS )zE Try to get current process ready to unpickle process object r! rA r@ rB rC rD rF rE rG T)ZforcerI rK N)r r7 r! rA r r@ Z get_loggerZsetLevelr rM r r0 �chdirrQ r �_fixup_main_from_name�_fixup_main_from_path)�datar r r r; � s, r; c C s~ t jd }| dks | jd�r d S t|jdd �| kr6d S tj|� tjd�}t j | ddd�}|j j |� | t jd< t jd< d S )NrH z .__main__r! � __mp_main__T)�run_nameZ alter_sys) r rT �endswithr' rU �old_main_modules�append�types� ModuleType�runpyZ run_module�__dict__rR )Zmod_name� current_mainr\ � main_contentr r r r_ � s r_ c C s� t jd }tjjtjj| ��d }|dkr.d S t|dd �| krBd S tj|� t j d�}t j | dd�}|j j|� | t jd< t jd< d S )NrH r ZipythonrJ rb )rc )r rT r0 rM �splitext�basenamer' re rf rg rh ri Zrun_pathrj rR )r] rk Z main_namer\ rl r r r r` � s r` c C s t | � dS )z< Set sys.modules['__main__'] to module at main_path N)r` )r] r r r r s )NN)%r0 r ri rg r2 r r r �contextr r �__all__r. rV rW r' r( �lowerrd rM r* � exec_prefixr r r r r r r r r? r re r; r_ r` r r r r r �<module> s> 2& __pycache__/pool.cpython-36.opt-2.pyc 0000644 00000045034 15204243006 0013315 0 ustar 00 3 \�e � @ s8 d dgZ ddlZddlZddlZddlZddlZddlZddlZddlm Z ddlm Z m Z dZ dZ dZej� Zdd � Zd d � ZG d d � d e�ZG dd� d�Zdd� ZG dd� de�Zdf ddfdd�Zdd� ZG dd � d e�ZG dd� de�Z e Z G d d � d e �Z G d d � d e�Z G d d!� d!e �Z G d"d� de�Z!dS )#�Pool� ThreadPool� N� )�util)� get_context� TimeoutError� c C s t t| � �S )N)�list�map)�args� r �,/usr/lib64/python3.6/multiprocessing/pool.py�mapstar+ s r c C s t tj| d | d ��S )Nr r )r � itertools�starmap)r r r r � starmapstar. s r c @ s e Zd Zdd� Zdd� ZdS )�RemoteTracebackc C s || _ d S )N)�tb)�selfr r r r �__init__6 s zRemoteTraceback.__init__c C s | j S )N)r )r r r r �__str__8 s zRemoteTraceback.__str__N)�__name__� __module__� __qualname__r r r r r r r 5 s r c @ s e Zd Zdd� Zdd� ZdS )�ExceptionWithTracebackc C s0 t jt|�||�}dj|�}|| _d| | _d S )N� z """ %s""")� traceback�format_exception�type�join�excr )r r r r r r r < s z ExceptionWithTraceback.__init__c C s t | j| jffS )N)� rebuild_excr r )r r r r � __reduce__A s z!ExceptionWithTraceback.__reduce__N)r r r r r" r r r r r ; s r c C s t |�| _| S )N)r � __cause__)r r r r r r! D s r! c s, e Zd Z� fdd�Zdd� Zdd� Z� ZS )�MaybeEncodingErrorc s. t |�| _t |�| _tt| �j| j| j� d S )N)�reprr �value�superr$ r )r r r& )� __class__r r r P s zMaybeEncodingError.__init__c C s d| j | jf S )Nz(Error sending result: '%s'. Reason: '%s')r& r )r r r r r U s zMaybeEncodingError.__str__c C s d| j j| f S )Nz<%s: %s>)r( r )r r r r �__repr__Y s zMaybeEncodingError.__repr__)r r r r r r) � __classcell__r r )r( r r$ L s r$ Fc ' C s� |j }| j}t| d�r*| jj� |jj� |d k r:||� d}�xH|d ksX|oT||k �r�y |� } W n$ ttfk r� tj d� P Y nX | d kr�tj d� P | \} } } } }yd| | |�f}W nB t k r� } z&|r�| t k r�t ||j �}d|f}W Y d d }~X nX y|| | |f� W nR t k �rd } z4t||d �}tj d| � || | d|ff� W Y d d }~X nX d } } } } } }|d7 }qBW tj d | � d S ) N�_writerr z)worker got EOFError or OSError -- exitingz worker got sentinel -- exitingTFr z0Possible encoding error while sending result: %sz worker exiting after %d tasks)�put�get�hasattrr+ �close�_reader�EOFError�OSErrorr �debug� Exception�_helper_reraises_exceptionr � __traceback__r$ )�inqueue�outqueue� initializer�initargsZmaxtasksZwrap_exceptionr, r- Z completed�task�job�i�funcr �kwds�result�e�wrappedr r r �worker] sD $ rC c C s | �d S )Nr )Zexr r r r5 � s r5 c @ s2 e Zd ZdZdd� Zddf ddfdd�Zdd� Zd d � Zd d � Zd d� Z f i fdd�Z d<dd�Z d=dd�Z d>dd�Z dd� Zd?dd�Zd@d d �Zf i ddfd d �ZdAd d!�ZdBd"d#�Zed$d%� �Zed&d'� �Zed(d)� �Zed*d+� �Zd,d-� Zd.d/� Zd0d1� Zd2d3� Z ed4d5� �Z e d6d7� �Z d8d9� Z d:d;� Z!dS )Cr Tc O s | j j||�S )N)�_ctx�Process)r r r? r r r rE � s z Pool.ProcessNc C sn |pt � | _| j� tj� | _i | _t| _|| _ || _ || _ |d krPt j � pNd}|dk r`td��|d k rzt|� rztd��|| _g | _| j� tjtj| fd�| _d| j_t| j_| jj� tjtj| j| j | j | j| jfd�| _ d| j _t| j _| j j� tjtj | j | j | jfd�| _!d| j!_t| j!_| j!j� t"j#| | j$| j| j%| j | j| j| j | j!| jfdd�| _&d S )Nr z&Number of processes must be at least 1z initializer must be a callable)�targetr T� )r Z exitpriority)'r rD � _setup_queues�queue�Queue� _taskqueue�_cache�RUN�_state�_maxtasksperchild� _initializer� _initargs�os� cpu_count� ValueError�callable� TypeError� _processes�_pool�_repopulate_pool� threadingZThreadr �_handle_workers�_worker_handler�daemon�start� _handle_tasks� _quick_put� _outqueue� _task_handler�_handle_results� _quick_get�_result_handlerr ZFinalize�_terminate_pool�_inqueue� _terminate)r � processesr9 r: Zmaxtasksperchild�contextr r r r � sT z Pool.__init__c C sZ d}xPt tt| j���D ]:}| j| }|jd k rtjd| � |j� d}| j|= qW |S )NFzcleaning up worker %dT)�reversed�range�lenrX �exitcoder r3 r )r Zcleanedr= rC r r r �_join_exited_workers� s zPool._join_exited_workersc C s| xvt | jt| j� �D ]^}| jt| j| j| j| j | j | j fd�}| jj |� |j jdd�|_ d|_|j� tjd� qW d S )N)rF r rE Z PoolWorkerTz added worker)rl rW rm rX rE rC rg ra rP rQ rO �_wrap_exception�append�name�replacer] r^ r r3 )r r= �wr r r rY � s zPool._repopulate_poolc C s | j � r| j� d S )N)ro rY )r r r r �_maintain_pool� s zPool._maintain_poolc C s4 | j j� | _| j j� | _| jjj| _| jjj| _ d S )N) rD Z SimpleQueuerg ra r+ �sendr` r0 �recvrd )r r r r rH � s zPool._setup_queuesc C s | j |||�j� S )N)� apply_asyncr- )r r> r r? r r r �apply� s z Pool.applyc C s | j ||t|�j� S )N)� _map_asyncr r- )r r> �iterable� chunksizer r r r s zPool.mapc C s | j ||t|�j� S )N)rz r r- )r r> r{ r| r r r r s z Pool.starmapc C s | j ||t|||�S )N)rz r )r r> r{ r| �callback�error_callbackr r r � starmap_async s zPool.starmap_asyncc c sn y0d}x&t |�D ]\}}||||fi fV qW W n8 tk rh } z ||d t|fi fV W Y d d }~X nX d S )Nr ���)� enumerater4 r5 )r Z result_jobr> r{ r= �xrA r r r �_guarded_task_generation s z Pool._guarded_task_generationr c C s� | j tkrtd��|dkrFt| j�}| jj| j|j||�|j f� |S t j |||�}t| j�}| jj| j|jt |�|j f� dd� |D �S d S )NzPool not runningr c s s | ]}|D ] }|V q qd S )Nr )�.0�chunk�itemr r r � <genexpr>@ s z Pool.imap.<locals>.<genexpr>) rN rM rT � IMapIteratorrL rK r, r� �_job� _set_lengthr � _get_tasksr )r r> r{ r| r@ � task_batchesr r r �imap' s z Pool.imapc C s� | j tkrtd��|dkrFt| j�}| jj| j|j||�|j f� |S t j |||�}t| j�}| jj| j|jt |�|j f� dd� |D �S d S )NzPool not runningr c s s | ]}|D ] }|V q qd S )Nr )r� r� r� r r r r� [ s z&Pool.imap_unordered.<locals>.<genexpr>) rN rM rT �IMapUnorderedIteratorrL rK r, r� r� r� r r� r )r r> r{ r| r@ r� r r r �imap_unorderedB s zPool.imap_unorderedc C sB | j tkrtd��t| j||�}| jj|jd|||fgd f� |S )NzPool not runningr )rN rM rT � ApplyResultrL rK r, r� )r r> r r? r} r~ r@ r r r rx ] s zPool.apply_asyncc C s | j ||t|||�S )N)rz r )r r> r{ r| r} r~ r r r � map_asynch s zPool.map_asyncc C s� | j tkrtd��t|d�s$t|�}|d krTtt|�t| j�d �\}}|rT|d7 }t|�dkrdd}tj |||�}t | j |t|�||d�} | j j | j| j||�d f� | S )NzPool not running�__len__� r r )r~ )rN rM rT r. r �divmodrm rX r r� � MapResultrL rK r, r� r� ) r r> r{ Zmapperr| r} r~ Zextrar� r@ r r r rz p s& zPool._map_asyncc C sT t j� }x0|jtks$| jr8|jtkr8| j� tjd� q W | j j d � t j d� d S )Ng�������?zworker handler exiting) rZ �current_threadrN rM rL � TERMINATEru �time�sleeprK r, r r3 )�pool�threadr r r r[ � s zPool._handle_workersc C sj t j� }x�t| jd �D ]�\}}d }z�x�|D ]�}|jr@tjd� P y ||� W q* tk r� } zD|d d� \} } y|| j| d| f� W n t k r� Y nX W Y d d } ~ X q*X q*W |r�tjd� |r�|d nd } || d � wP W d d } }} X qW tjd� y:tjd� |j d � tjd� x|D ]} |d � �q&W W n t k �rZ tjd � Y nX tjd � d S ) Nz'task handler found thread._state != RUNr Fzdoing set_length()r ztask handler got sentinelz/task handler sending sentinel to result handlerz(task handler sending sentinel to workersz/task handler got OSError when sending sentinelsztask handler exitingr� ) rZ r� �iterr- rN r r3 r4 �_set�KeyErrorr, r2 ) � taskqueuer, r8 r� �cacher� ZtaskseqZ set_lengthr; rA r< �idx�pr r r r_ � sB zPool._handle_tasksc C s� t j� }x�y |� }W n" ttfk r6 tjd� d S X |jrJtjd� P |d kr^tjd� P |\}}}y|| j||� W n tk r� Y nX d } }}q W x�|o�|jt k�r:y |� }W n" ttfk r� tjd� d S X |d kr�tjd� q�|\}}}y|| j||� W n tk �r* Y nX d } }}q�W t | d��r�tjd� y,x&t d�D ]}| j j � �spP |� �q^W W n ttfk �r� Y nX tjdt|�|j� d S ) Nz.result handler got EOFError/OSError -- exitingz,result handler found thread._state=TERMINATEzresult handler got sentinelz&result handler ignoring extra sentinelr0 z"ensuring that outqueue is not full� z7result handler exiting: len(cache)=%s, thread._state=%s)rZ r� r2 r1 r r3 rN r� r� r� r. rl r0 �pollrm )r8 r- r� r� r; r< r= �objr r r rc � sZ zPool._handle_resultsc c s4 t |�}x&ttj||��}|s"d S | |fV q W d S )N)r� �tupler �islice)r> �it�sizer� r r r r� s zPool._get_tasksc C s t d��d S )Nz:pool objects cannot be passed between processes or pickled)�NotImplementedError)r r r r r" s zPool.__reduce__c C s& t jd� | jtkr"t| _t| j_d S )Nz closing pool)r r3 rN rM �CLOSEr\ )r r r r r/ s z Pool.closec C s$ t jd� t| _t| j_| j� d S )Nzterminating pool)r r3 r� rN r\ rh )r r r r � terminate s zPool.terminatec C sD t jd� | jj� | jj� | jj� x| jD ] }|j� q0W d S )Nz joining pool)r r3 r\ r rb re rX )r r� r r r r s z Pool.joinc C sD t jd� | jj� x*|j� r>| jj� r>| jj� tj d� qW d S )Nz7removing tasks from inqueue until task handler finishedr ) r r3 Z_rlock�acquire�is_aliver0 r� rw r� r� )r7 � task_handlerr� r r r �_help_stuff_finish( s zPool._help_stuff_finishc C s( t jd� t|_t|_t jd� | j||t|�� t|_|jd � t jd� tj� |k r`|j � |r�t |d d�r�t jd� x |D ]} | j d kr�| j � q�W t jd� tj� |k r�|j � t jd� tj� |k r�|j � |o�t |d d��r$t jd � x,|D ]$} | j � r�t jd | j � | j � q�W d S ) Nzfinalizing poolz&helping task handler/workers to finishzjoining worker handlerr r� zterminating workerszjoining task handlerzjoining result handlerzjoining pool workerszcleaning up worker %d)r r3 r� rN r� rm r, rZ r� r r. rn r� r� �pid) �clsr� r7 r8 r� Zworker_handlerr� Zresult_handlerr� r� r r r rf 1 s6 zPool._terminate_poolc C s | S )Nr )r r r r � __enter___ s zPool.__enter__c C s | j � d S )N)r� )r �exc_typeZexc_valZexc_tbr r r �__exit__b s z Pool.__exit__)N)N)NNN)r )r )NNN)NNN)"r r r rp rE r ro rY ru rH ry r r r r� r� r� rx r� rz � staticmethodr[ r_ rc r� r" r/ r� r r� � classmethodrf r� r� r r r r r � sD 8 . < .c @ s@ e Zd Zdd� Zdd� Zdd� Zddd �Zdd d �Zd d � ZdS )r� c C s4 t j� | _tt�| _|| _|| _|| _| || j< d S )N) rZ ZEvent�_event�next� job_counterr� rL � _callback�_error_callback)r r� r} r~ r r r r k s zApplyResult.__init__c C s | j j� S )N)r� Zis_set)r r r r �readys s zApplyResult.readyc C s | j S )N)�_success)r r r r � successfulv s zApplyResult.successfulNc C s | j j|� d S )N)r� �wait)r �timeoutr r r r� z s zApplyResult.waitc C s, | j |� | j� st�| jr"| jS | j�d S )N)r� r� r r� �_value)r r� r r r r- } s zApplyResult.getc C sV |\| _ | _| jr$| j r$| j| j� | jr>| j r>| j| j� | jj� | j| j= d S )N)r� r� r� r� r� �setrL r� )r r= r� r r r r� � s zApplyResult._set)N)N) r r r r r� r� r� r- r� r r r r r� i s r� c @ s e Zd Zdd� Zdd� ZdS )r� c C sf t j| |||d� d| _d g| | _|| _|dkrLd| _| jj� || j= n|| t || � | _d S )N)r~ Tr ) r� r r� r� � _chunksize� _number_leftr� r� r� �bool)r r� r| �lengthr} r~ r r r r � s zMapResult.__init__c C s� | j d8 _ |\}}|rp| jrp|| j|| j |d | j �< | j dkr�| jrZ| j| j� | j| j= | jj� nH| r�| jr�d| _|| _| j dkr�| j r�| j | j� | j| j= | jj� d S )Nr r F) r� r� r� r� r� rL r� r� r� r� )r r= Zsuccess_result�successr@ r r r r� � s" zMapResult._setN)r r r r r� r r r r r� � s r� c @ s: e Zd Zdd� Zdd� Zd dd�ZeZdd � Zd d � ZdS ) r� c C sJ t jt j� �| _tt�| _|| _tj � | _ d| _ d | _ i | _ | || j< d S )Nr )rZ Z ConditionZLock�_condr� r� r� rL � collections�deque�_items�_index�_length� _unsorted)r r� r r r r � s zIMapIterator.__init__c C s | S )Nr )r r r r �__iter__� s zIMapIterator.__iter__Nc C s� | j �~ y| jj� }W nh tk r~ | j| jkr6t�| j j|� y| jj� }W n( tk rx | j| jkrpt�t�Y nX Y nX W d Q R X |\}}|r�|S |�d S )N) r� r� �popleft� IndexErrorr� r� � StopIterationr� r )r r� r� r� r& r r r r� � s" zIMapIterator.nextc C s� | j �� | j|krr| jj|� | jd7 _x8| j| jkrd| jj| j�}| jj|� | jd7 _q.W | j j� n || j|< | j| jkr�| j| j = W d Q R X d S )Nr ) r� r� r� rq r� �pop�notifyr� rL r� )r r= r� r r r r� � s zIMapIterator._setc C s< | j �, || _| j| jkr.| j j� | j| j= W d Q R X d S )N)r� r� r� r� rL r� )r r� r r r r� � s zIMapIterator._set_length)N) r r r r r� r� �__next__r� r� r r r r r� � s r� c @ s e Zd Zdd� ZdS )r� c C sP | j �@ | jj|� | jd7 _| j j� | j| jkrB| j| j= W d Q R X d S )Nr )r� r� rq r� r� r� rL r� )r r= r� r r r r� s zIMapUnorderedIterator._setN)r r r r� r r r r r� � s r� c @ s@ e Zd ZdZedd� �Zddf fdd�Zdd� Zed d � �ZdS ) r Fc O s ddl m} || |�S )Nr )rE )ZdummyrE )r r? rE r r r rE s zThreadPool.ProcessNc C s t j| |||� d S )N)r r )r ri r9 r: r r r r s zThreadPool.__init__c C s, t j� | _t j� | _| jj| _| jj| _d S )N)rI rJ rg ra r, r` r- rd )r r r r rH s zThreadPool._setup_queuesc C s< | j �, | jj� | jjd g| � | j j� W d Q R X d S )N)Z not_emptyrI �clear�extendZ notify_all)r7 r� r� r r r r� s z ThreadPool._help_stuff_finish) r r r rp r� rE r rH r� r r r r r s )"�__all__rZ rI r r� rR r� r r r r r rM r� r� �countr� r r r4 r r r! r$ rC r5 �objectr r� Z AsyncResultr� r� r� r r r r r �<module> s@ * \&)@ __pycache__/popen_spawn_posix.cpython-36.opt-1.pyc 0000644 00000004074 15204243006 0016115 0 ustar 00 3 \p � @ sp d dl Z d dlZddlmZmZ ddlmZ ddlmZ ddlmZ dgZ G dd � d e �Z G d d� dej �Z dS ) � N� )� reduction�set_spawning_popen)� popen_fork)�spawn)�util�Popenc @ s e Zd Zdd� Zdd� ZdS )�_DupFdc C s || _ d S )N)�fd)�selfr � r �9/usr/lib64/python3.6/multiprocessing/popen_spawn_posix.py�__init__ s z_DupFd.__init__c C s | j S )N)r )r r r r �detach s z _DupFd.detachN)�__name__� __module__� __qualname__r r r r r r r s r c s4 e Zd ZdZeZ� fdd�Zdd� Zdd� Z� Z S )r r c s g | _ t� j|� d S )N)�_fds�superr )r � process_obj)� __class__r r r s zPopen.__init__c C s | j j|� |S )N)r �append)r r r r r �duplicate_for_child" s zPopen.duplicate_for_childc C s@ ddl m} |j� }| jj|� tj|j�}tj � }t | � z t j ||� t j ||� W d t d � X d } } }} z~t j� \}}t j� \}} tj||d�} | jj||g� tjtj� | | j�| _|| _t| ddd��} | j|j� � W d Q R X W d |d k �rtj| t j|f� x(||| fD ]} | d k �r t j| � �q W X d S )Nr )�semaphore_tracker)� tracker_fdZ pipe_handle�wbF)�closefd)� r Zgetfdr r r Zget_preparation_data�_name�io�BytesIOr r �dump�os�pipeZget_command_line�extendr Zspawnv_passfdsZget_executable�pid�sentinel�open�write� getbufferZFinalize�close) r r r r Z prep_data�fpZparent_rZchild_wZchild_rZparent_w�cmd�fr r r r �_launch& s6 z Popen._launch) r r r �methodr ZDupFdr r r. � __classcell__r r )r r r s ) r r" �contextr r r r r r �__all__�objectr r r r r r �<module> s __pycache__/semaphore_tracker.cpython-36.opt-1.pyc 0000644 00000007065 15204243006 0016043 0 ustar 00 3 \ � @ s� d dl Z d dlZd dlZd dlZd dlZd dlZddlmZ ddlmZ dddgZ G dd � d e �Z e � Z e j Z e jZe jZe jZd d � ZdS ) � N� )�spawn)�util�ensure_running�register� unregisterc @ s< e Zd Zdd� Zdd� Zdd� Zdd� Zd d � Zd d � Zd S )�SemaphoreTrackerc C s t j� | _d | _d | _d S )N)� threadingZLock�_lock�_fd�_pid)�self� r �9/usr/lib64/python3.6/multiprocessing/semaphore_tracker.py�__init__ s zSemaphoreTracker.__init__c C s | j � | jS )N)r r )r r r r �getfd" s zSemaphoreTracker.getfdc C s | j �� | jdk rRtj| jtj�\}}|s0dS tj| j� d| _d| _tjd� g }y|j t j j � � W n t k r~ Y nX d}tj� \}}zlyB|j |� tj� }|gtj� }|d|| g7 }tj|||�}W n tj|� � Y nX || _|| _W dtj|� X W dQ R X dS )z�Make sure that semaphore tracker process is running. This can be run from any process. Usually a child process will use the semaphore created by its parent.NzWsemaphore_tracker: process died unexpectedly, relaunching. Some semaphores might leak.z;from multiprocessing.semaphore_tracker import main;main(%d)z-c)r r �os�waitpid�WNOHANG�closer �warnings�warn�append�sys�stderr�fileno� Exception�piper Zget_executabler Z _args_from_interpreter_flagsZspawnv_passfds) r �pidZstatusZ fds_to_pass�cmd�r�wZexe�argsr r r r & s8 z SemaphoreTracker.ensure_runningc C s | j d|� dS )z2Register name of semaphore with semaphore tracker.ZREGISTERN)�_send)r �namer r r r Q s zSemaphoreTracker.registerc C s | j d|� dS )z4Unregister name of semaphore with semaphore tracker.Z UNREGISTERN)r# )r r$ r r r r U s zSemaphoreTracker.unregisterc C s@ | j � dj||�jd�}t|�dkr.td��tj| j|�}d S )Nz{0}:{1} �asciii z name too long)r �format�encode�len� ValueErrorr �writer )r r r$ �msg�nbytesr r r r# Y s zSemaphoreTracker._sendN) �__name__� __module__� __qualname__r r r r r r# r r r r r s +r c - C s� t j t jt j� t j t jt j� x6tjtjfD ]&}y |j� W q. tk rR Y q.X q.W t � }z�t | d���}x�|D ]�}yJ|j � j d�\}}|dkr�|j |� n |dkr�|j|� n td| ��W qr tk r� ytjtj� � W n Y nX Y qrX qrW W dQ R X W d|�r<ytjdt|� � W n tk �r: Y nX xf|D ]^}zT|jd�}ytj|� W n6 tk �r� } ztjd ||f � W Y dd}~X nX W dX �qBW X dS ) zRun semaphore tracker.�rb� :s REGISTERs UNREGISTERzunrecognized command %rNzRsemaphore_tracker: There appear to be %d leaked semaphores to clean up at shutdownr% zsemaphore_tracker: %r: %s)�signal�SIGINT�SIG_IGN�SIGTERMr �stdin�stdoutr r �set�open�strip�split�add�remove� RuntimeError� excepthook�exc_infor r r( �decode�_multiprocessingZ sem_unlink)�fd�f�cache�liner r$ �er r r �maink sJ (rH )r r2 r r r rB � r r �__all__�objectr Z_semaphore_trackerr r r r rH r r r r �<module> s I __pycache__/popen_forkserver.cpython-36.pyc 0000644 00000004430 15204243006 0014770 0 ustar 00 3 \� � @ s� d dl Z d dlZddlmZmZ ejs.ed��ddlmZ ddlm Z ddlm Z ddlm Z d gZ G d d � d e �ZG d d � d e j�ZdS ) � N� )� reduction�set_spawning_popenz,No support for sending fds between processes)� forkserver)� popen_fork)�spawn)�util�Popenc @ s e Zd Zdd� Zdd� ZdS )�_DupFdc C s || _ d S )N)�ind)�selfr � r �8/usr/lib64/python3.6/multiprocessing/popen_forkserver.py�__init__ s z_DupFd.__init__c C s t j� | j S )N)r Zget_inherited_fdsr )r r r r �detach s z _DupFd.detachN)�__name__� __module__� __qualname__r r r r r r r s r c sB e Zd ZdZeZ� fdd�Zdd� Zdd� Ze j fdd �Z � Z S ) r r c s g | _ t� j|� d S )N)�_fds�superr )r � process_obj)� __class__r r r ! s zPopen.__init__c C s | j j|� t| j �d S )Nr )r �append�len)r �fdr r r �duplicate_for_child% s zPopen.duplicate_for_childc C s� t j|j�}tj� }t| � z tj||� tj||� W d td � X tj | j �\| _ }t j | tj| j f� t|ddd��}|j|j� � W d Q R X tj| j �| _d S )N�wbT)�closefd)r Zget_preparation_data�_name�io�BytesIOr r �dumpr Zconnect_to_new_processr �sentinelr ZFinalize�os�close�open�write� getbuffer� read_unsigned�pid)r r Z prep_dataZbuf�w�fr r r �_launch) s z Popen._launchc C sr | j d krlddlm} |tjkr$dnd }|| jg|�s:d S ytj| j�| _ W n tt fk rj d| _ Y nX | j S )Nr )�wait� ) � returncodeZmultiprocessing.connectionr- r# �WNOHANGr" r r( �OSError�EOFError)r �flagr- Ztimeoutr r r �poll9 s z Popen.poll) r r r �methodr ZDupFdr r r, r# r0 r4 � __classcell__r r )r r r s )r r# �contextr r ZHAVE_SEND_HANDLE� ImportError� r r r r �__all__�objectr r r r r r �<module> s __pycache__/resource_sharer.cpython-36.opt-2.pyc 0000644 00000011130 15204243006 0015525 0 ustar 00 3 \� � @ s� d dl Z d dlZd dlZd dlZd dlZddlmZ ddlmZ ddlm Z dgZ ej dkrxe dg7 Z G d d� de �Z ne d g7 Z G d d � d e �ZG d d � d e �Ze� ZejZdS )� N� )�process)� reduction)�util�stopZwin32� DupSocketc @ s e Zd Zdd� Zdd� ZdS )r c s( |j � � � fdd�}tj|� j�| _d S )Nc s � j |�}| j|� d S )N)�shareZ send_bytes)�conn�pidr )�new_sock� �7/usr/lib64/python3.6/multiprocessing/resource_sharer.py�send s z DupSocket.__init__.<locals>.send)�dup�_resource_sharer�register�close�_id)�selfZsockr r )r r �__init__ s zDupSocket.__init__c C s* t j| j��}|j� }tj|�S Q R X d S )N)r �get_connectionr Z recv_bytes�socketZ fromshare)r r r r r r �detach$ s zDupSocket.detachN)�__name__� __module__� __qualname__r r r r r r r s �DupFdc @ s e Zd Zdd� Zdd� ZdS )r c s4 t j|�� � fdd�}� fdd�}tj||�| _d S )Nc s t j| � |� d S )N)r Z send_handle)r r )�new_fdr r r 1 s z DupFd.__init__.<locals>.sendc s t j� � d S )N)�osr r )r r r r 3 s z DupFd.__init__.<locals>.close)r r r r r )r �fdr r r )r r r / s zDupFd.__init__c C s" t j| j�� }tj|�S Q R X d S )N)r r r r Z recv_handle)r r r r r r 7 s z DupFd.detachN)r r r r r r r r r r - s c @ sJ e Zd Zdd� Zdd� Zedd� �Zddd �Zd d � Zd d � Z dd� Z dS )�_ResourceSharerc C s@ d| _ i | _g | _tj� | _d | _d | _d | _t j | t j � d S )Nr ) �_key�_cache� _old_locks� threading�Lock�_lock� _listener�_address�_threadr Zregister_after_forkr � _afterfork)r r r r r ? s z_ResourceSharer.__init__c C sN | j �> | jd kr| j� | jd7 _||f| j| j< | j| jfS Q R X d S )Nr )r& r( �_startr! r" )r r r r r r r I s z_ResourceSharer.registerc C s< ddl m} | \}}||tj� jd�}|j|tj� f� |S )Nr )�Client)�authkey)� connectionr, r �current_processr- r r �getpid)Zidentr, �address�key�cr r r r R s z _ResourceSharer.get_connectionNc C s� ddl m} | j�� | jd k r�|| jtj� jd�}|jd � |j� | j j |� | j j � rdt j d� | jj� d | _ d | _d | _x | jj� D ]\}\}}|� q�W | jj� W d Q R X d S )Nr )r, )r- z._ResourceSharer thread did not stop when asked)r. r, r& r( r r/ r- r r r) �joinZis_aliver Z sub_warningr' r" �items�clear)r Ztimeoutr, r3 r2 r r r r r r [ s" z_ResourceSharer.stopc C sn x | j j� D ]\}\}}|� q W | j j� | jj| j� tj� | _| jd k rX| jj � d | _d | _ d | _ d S )N) r" r5 r6 r# �appendr& r$ r% r' r r( r) )r r2 r r r r r r* p s z_ResourceSharer._afterforkc C sX ddl m} tjd� |tj� jd�| _| jj| _ t j | j d�}d|_ |j� || _d S )Nr )�Listenerz0starting listener and thread for sending handles)r- )�targetT)r. r8 r �debugr r/ r- r' r1 r( r$ ZThread�_serveZdaemon�startr) )r r8 �tr r r r+ ~ s z_ResourceSharer._startc C s� t td�r tjtjtdtj�� x�y\| jj� �H}|j� }|d krBP |\}}| j j |�\}}z|||� W d |� X W d Q R X W q" t j � s�t jt j� � Y q"X q"W d S )N�pthread_sigmaskr )�hasattr�signalr> � SIG_BLOCK�range�NSIGr' ZacceptZrecvr" �popr Z is_exiting�sys� excepthook�exc_info)r r �msgr2 Zdestination_pidr r r r r r; � s z_ResourceSharer._serve)N) r r r r r � staticmethodr r r* r+ r; r r r r r = s r )r r@ r rE r$ � r �contextr r �__all__�platform�objectr r r r r r r r r �<module> s ` __pycache__/util.cpython-36.opt-2.pyc 0000644 00000021432 15204243006 0013315 0 ustar 00 3 \n. � @ s� d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ ddlm Z ddddd d d d d ddddddgZ d Z dZ dZ dZdZdZdZdadadd� Zd d� Zd d� Zd d� Zd d � Zd6d d �Zd!d � Zej� Z ej � Z d"d#� Z d$d � Z i Z!ej � Z"G d%d� de#�Z$d7d&d'�Z%d(d � Z&da'eee%e j(e j)fd)d*�Z*ej+e*� G d+d� de#�Z,G d,d� dej-�Z.ye j/d-�Z0W n e1k �r� d.Z0Y nX d/d� Z2d0d1� Z3d2d3� Z4d4d5� Z5dS )8� N)� _args_from_interpreter_flags� )�process� sub_debug�debug�info� sub_warning� get_logger� log_to_stderr� get_temp_dir�register_after_fork� is_exiting�Finalize�ForkAwareThreadLock�ForkAwareLocal�close_all_fds_except�SUBDEBUG� SUBWARNING� � � � Zmultiprocessingz+[%(levelname)s/%(processName)s] %(message)sFc G s t rt jt| f|�� d S )N)�_logger�logr )�msg�args� r �,/usr/lib64/python3.6/multiprocessing/util.pyr , s c G s t rt jt| f|�� d S )N)r r �DEBUG)r r r r r r 0 s c G s t rt jt| f|�� d S )N)r r �INFO)r r r r r r 4 s c G s t rt jt| f|�� d S )N)r r r )r r r r r r 8 s c C s| dd l } | j� z\tsj| jt�adt_ttd�rFtjt � tj t � n$tj j t f i f� tj j t f i f� W d | j� X tS )Nr � unregister)�loggingZ _acquireLockr Z getLogger� LOGGER_NAMEZ propagate�hasattr�atexitr �_exit_function�registerZ _exithandlers�remove�appendZ _releaseLock)r! r r r r < s c C sJ dd l }t� }|jt�}|j� }|j|� |j|� | rB|j| � dat S )Nr T) r! r Z Formatter�DEFAULT_LOGGING_FORMATZ StreamHandlerZ setFormatterZ addHandlerZsetLevel�_log_to_stderrr )�levelr! ZloggerZ formatterZhandlerr r r r W s c C sd t j� jjd�} | d kr`dd l}dd l}|jdd�} td| � td |j | gdd� | t j� jd< | S ) N�tempdirr zpymp-)�prefixzcreated temp directory %s�d )r � exitpriorityi����) r �current_processZ_config�get�shutil�tempfileZmkdtempr r Zrmtree)r, r2 r3 r r r r m s c C sj t tj� �} | j� xP| D ]H\\}}}}y ||� W q tk r` } ztd|� W Y d d }~X qX qW d S )Nz after forker raised exception %s)�list�_afterfork_registry�items�sort� Exceptionr )r6 �indexZident�func�obj�er r r �_run_after_forkers s r= c C s | t tt�t| �|f< d S )N)r5 �next�_afterfork_counter�id)r; r: r r r r � s c @ sH e Zd Zf ddfdd�Zdeeejfdd�Zdd� Z dd � Z d d � Z dS ) r Nc C sT |d k rt j|| �| _n || _|| _|p*i | _|tt�f| _t j � | _ | t | j< d S )N) �weakref�ref�_weakref� _callback�_args�_kwargsr> �_finalizer_counter�_key�os�getpid�_pid�_finalizer_registry)�selfr; �callbackr �kwargsr/ r r r �__init__� s zFinalize.__init__c C s� y || j = W n tk r( |d� Y nbX | j|� krD|d� d }n$|d| j| j| j� | j| j| j�}d | _ | _ | _ | _| _ |S d S )Nz finalizer no longer registeredz+finalizer ignored because different processz/finalizer calling %s with args %s and kwargs %s)rH �KeyErrorrK rD rE rF rC )rM �wrrL r rJ �resr r r �__call__� s zFinalize.__call__c C sD y t | j= W n tk r Y n X d | _ | _ | _ | _| _d S )N)rL rH rQ rC rD rE rF )rM r r r �cancel� s zFinalize.cancelc C s | j tkS )N)rH rL )rM r r r � still_active� s zFinalize.still_activec C s� y | j � }W n ttfk r( d }Y nX |d kr>d| jj S d| jjt| jd| j�f }| jrr|dt| j� 7 }| j r�|dt| j � 7 }| j d d k r�|dt| j d � 7 }|d S ) Nz<%s object, dead>z<%s object, callback=%s�__name__z, args=z , kwargs=r z, exitprority=�>) rC �AttributeError� TypeError� __class__rW �getattrrD rE �strrF rH )rM r; �xr r r �__repr__� s zFinalize.__repr__) rW � __module__� __qualname__rP rL r rI rJ rT rU rV r_ r r r r r � s c s� t d kr d S �d kr dd� � n �fdd�� � fdd�tt �D �}|jdd� xX|D ]P}t j|�}|d k rRtd|� y |� W qR tk r� d d l}|j� Y qRX qRW �d kr�t j� d S ) Nc S s | d d k S )Nr r )�pr r r �<lambda>� s z!_run_finalizers.<locals>.<lambda>c s | d d k o| d � kS )Nr r )rb )� minpriorityr r rc � s c s g | ]}� |�r|�qS r r )�.0�key)�fr r � <listcomp>� s z#_run_finalizers.<locals>.<listcomp>T)�reversez calling %sr ) rL r4 r7 r1 r r8 � traceback� print_exc�clear)rd �keysrf � finalizerrj r )rg rd r �_run_finalizers� s$ ro c C s t p t d kS )N)�_exitingr r r r r s c C s� t s�da | d� |d� |d� |� d k rzx*|� D ] }|jr2| d|j� |jj� q2W x"|� D ]}| d|j� |j� q^W |d� |� d S )NTzprocess shutting downz2running all "atexit" finalizers with priority >= 0r z!calling terminate() for daemon %sz calling join() for process %sz)running the remaining "atexit" finalizers)rp Zdaemon�nameZ_popenZ terminate�join)r r ro �active_childrenr0 rb r r r r% s r% c @ s, e Zd Zdd� Zdd� Zdd� Zdd� Zd S ) r c C s | j � t| tj � d S )N)�_resetr r )rM r r r rP K s z ForkAwareThreadLock.__init__c C s" t j� | _| jj| _| jj| _d S )N)� threadingZLock�_lock�acquire�release)rM r r r rt O s zForkAwareThreadLock._resetc C s | j j� S )N)rv � __enter__)rM r r r ry T s z ForkAwareThreadLock.__enter__c G s | j j|� S )N)rv �__exit__)rM r r r r rz W s z ForkAwareThreadLock.__exit__N)rW r` ra rP rt ry rz r r r r r J s c @ s e Zd Zdd� Zdd� ZdS )r c C s t | dd� � d S )Nc S s | j j� S )N)�__dict__rl )r; r r r rc ] s z)ForkAwareLocal.__init__.<locals>.<lambda>)r )rM r r r rP \ s zForkAwareLocal.__init__c C s t | �f fS )N)�type)rM r r r � __reduce__^ s zForkAwareLocal.__reduce__N)rW r` ra rP r} r r r r r [ s � SC_OPEN_MAX� c C sR t | �dtg } | j� x4tt| �d �D ] }tj| | d | |d � q*W d S )Nr ���)r4 �MAXFDr7 �range�lenrI � closerange)Zfds�ir r r r j s c C s� t jd krd S yt jj� W n ttfk r4 Y nX y@tjtjtj�} yt| dd�t _W n tj| � � Y nX W n ttfk r� Y nX d S )NF)�closefd) �sys�stdin�close�OSError� ValueErrorrI �open�devnull�O_RDONLY)�fdr r r � _close_stdint s r� c C sT yt jj� W n ttfk r& Y nX yt jj� W n ttfk rN Y nX d S )N)r� �stdout�flushrY r� �stderrr r r r �_flush_std_streams� s r� c C st dd l }tttt|���}tj� \}}z2|j|tj| �gd|d d ddddd d ||ddd �S tj |� tj |� X d S ) Nr Tr Fr� r� r� r� r� r� ) �_posixsubprocess�tuple�sorted�map�intrI �pipeZ fork_exec�fsencoder� )�pathr Zpassfdsr� Z errpipe_readZ errpipe_writer r r �spawnv_passfds� s r� )N)N)6rI � itertoolsr� rA r$ ru � subprocessr � r �__all__ZNOTSETr r r r r" r) r r* r r r r r r r �WeakValueDictionaryr5 �countr? r= r rL rG �objectr ro r rp rs r0 r% r& r Zlocalr �sysconfr� r8 r r� r� r� r r r r �<module> sf S ,( __pycache__/popen_fork.cpython-36.opt-1.pyc 0000644 00000004051 15204243006 0014477 0 ustar 00 3 \ � @ s> d dl Z d dlZd dlZddlmZ dgZG dd� de�ZdS )� N� )�util�Popenc @ sH e Zd ZdZdd� Zdd� Zejfdd�Zdd d �Z d d � Z d d� Z dS )r �forkc C s t j� d | _| j|� d S )N)r Z_flush_std_streams� returncode�_launch)�self� process_obj� r �2/usr/lib64/python3.6/multiprocessing/popen_fork.py�__init__ s zPopen.__init__c C s |S )Nr )r �fdr r r �duplicate_for_child s zPopen.duplicate_for_childc C s� | j d krzx>ytj| j|�\}}W n tk rB } zd S d }~X q X P q W || jkrztj|�rntj|� | _ n tj|�| _ | j S )N)r �os�waitpid�pid�OSError� WIFSIGNALED�WTERMSIG� WEXITSTATUS)r �flagr �sts�er r r �poll s z Popen.pollNc C sN | j d krH|d k r0ddlm} || jg|�s0d S | j|dkrBtjnd�S | j S )Nr )�waitg )r Zmultiprocessing.connectionr �sentinelr r �WNOHANG)r �timeoutr r r r r + s z Popen.waitc C s\ | j d krXytj| jtj� W n8 tk r2 Y n& tk rV | jdd�d krR� Y nX d S )Ng�������?)r ) r r �killr �signal�SIGTERM�ProcessLookupErrorr r )r r r r � terminate5 s zPopen.terminatec C s� d}t j� \}}t j� | _| jdkrdz0t j|� dtjkrJdd l}|j� |j � }W d t j |� X n"t j|� t j | t j|f� || _ d S )Nr r �random)r �piper r �close�sys�modulesr# Zseed� _bootstrap�_exitr ZFinalizer )r r �codeZparent_rZchild_wr# r r r r ? s z Popen._launch)N) �__name__� __module__� __qualname__�methodr r r r r r r" r r r r r r s )r r& r � r �__all__�objectr r r r r �<module> s __pycache__/process.cpython-36.pyc 0000644 00000017715 15204243006 0013067 0 ustar 00 3 \�# � @ s d ddgZ ddlZddlZddlZddlZddlmZ yejjej � �Z W n e k rb dZ Y nX dd� Z dd� Z dd � ZG d d � d e�ZG d d � d e�ZG d d� de�Ze� aejd�ae� a[i Zx<eejj � �D ]*\Z Z e dd� dko�de kr�e ee < q�W e� Z dS )� BaseProcess�current_process�active_children� N)�WeakSetc C s t S )z@ Return process object representing the current process )�_current_process� r r �//usr/lib64/python3.6/multiprocessing/process.pyr # s c C s t � tt�S )zN Return list of process objects corresponding to live child processes )�_cleanup�list� _childrenr r r r r ) s c C s. x(t t�D ] } | jj� d k r tj| � q W d S )N)r r �_popen�poll�discard)�pr r r r 4 s r c @ s� e Zd ZdZdd� Zdddf i fdd�dd�Zdd � Zd d � Zd d � Zd%dd�Z dd� Z e dd� �Z e j dd� �Z e dd� �Zej dd� �Ze dd� �Zej dd� �Ze dd � �Ze d d � �ZeZe d d � �Zd!d"� Zd#d$� ZdS )&r z� Process objects represent activity that is run in a separate process The class is analogous to `threading.Thread` c C s t �d S )N)�NotImplementedError)�selfr r r �_PopenD s zBaseProcess._PopenN)�daemonc C s� |d kst d��tt�}tj|f | _tjj� | _tj� | _ d | _ || _ t |�| _ t|�| _|p�t| �jd djdd� | jD �� | _|d k r�|| _tj| � d S )Nz#group argument must be None for now�-�:c s s | ]}t |�V qd S )N)�str)�.0�ir r r � <genexpr>S s z'BaseProcess.__init__.<locals>.<genexpr>)�AssertionError�next�_process_counterr � _identity�_config�copy�os�getpid� _parent_pidr �_target�tuple�_args�dict�_kwargs�type�__name__�join�_namer � _dangling�add)r �group�target�name�args�kwargsr �countr r r �__init__G s zBaseProcess.__init__c C s | j r| j | j| j� dS )zQ Method to be run in sub-process; can be overridden in sub-class N)r# r% r' )r r r r �runX s zBaseProcess.runc C st | j dkstd��| jtj� ks(td��tjjd� s>td��t� | j | �| _ | j j | _ | ` | ` | `tj| � dS )z% Start child process Nz cannot start a process twicez:can only start a process object created by current processr z3daemonic processes are not allowed to have children)r r r" r r! r r �getr r �sentinel� _sentinelr# r% r' r r- )r r r r �start_ s zBaseProcess.startc C s | j j� dS )zT Terminate process; sends SIGTERM signal or uses TerminateProcess() N)r � terminate)r r r r r: p s zBaseProcess.terminatec C sJ | j tj� kstd��| jdk s(td��| jj|�}|dk rFtj| � dS )z5 Wait until child process terminates z can only join a child processNz can only join a started process)r" r r! r r �waitr r )r Ztimeout�resr r r r* v s zBaseProcess.joinc C sX | t kr dS | jtj� ks"td��| jdkr0dS | jj� }|dkrFdS tj| � dS dS )z1 Return whether process is alive Tz can only test a child processNF) r r" r r! r r r r r )r � returncoder r r �is_alive� s zBaseProcess.is_alivec C s | j S )N)r+ )r r r r r0 � s zBaseProcess.namec C s t |t�std��|| _d S )Nzname must be a string)� isinstancer r r+ )r r0 r r r r0 � s c C s | j jdd�S )z4 Return whether process is a daemon r F)r r6 )r r r r r � s zBaseProcess.daemonc C s | j dkstd��|| jd< dS )z1 Set whether process is a daemon Nzprocess has already startedr )r r r )r Zdaemonicr r r r � s c C s | j d S )N�authkey)r )r r r r r@ � s zBaseProcess.authkeyc C s t |�| jd< dS )z2 Set authorization key of process r@ N)�AuthenticationStringr )r r@ r r r r@ � s c C s | j dkr| j S | j j� S )zM Return exit code of process or `None` if it has yet to stop N)r r )r r r r �exitcode� s zBaseProcess.exitcodec C s" | t krtj� S | jo | jjS dS )zU Return identifier (PID) of process or `None` if it has yet to start N)r r r! r �pid)r r r r �ident� s zBaseProcess.identc C s( y| j S tk r" td��Y nX dS )z{ Return a file descriptor (Unix) or handle (Windows) suitable for waiting for process termination. zprocess not startedN)r8 �AttributeError� ValueError)r r r r r7 � s zBaseProcess.sentinelc C s� | t krd}n>| jtj� kr"d}n*| jd kr2d}n| jj� d k rH| j}nd}t|�tkrv|dkrfd}ndt j ||� }dt| �j | j || j r�dp�d f S ) NZstarted�unknown�initialr Zstoppedz stopped[%s]z<%s(%s, %s%s)>z daemon� )r r" r r! r r rB r( �int�_exitcode_to_namer6 r) r+ r )r Zstatusr r r �__repr__� s zBaseProcess.__repr__c C sJ ddl m}m} �zy|| jd k r,|j| j� tjd�at� a |j � t }| a z|j j � |j� W d ~X |jd� z| j� d}W d |j� X W n� tk r� } zL|js�d}n:t|jd t�r�|jd }n tjjt|jd �d � d}W Y d d }~X n2 d}dd l}tjjd| j � |j � Y nX W d |jd| � |j � X |S )N� )�util�contextz child process calling self.run()r � z Process %s: z process exiting with exitcode %d) rI rN rO Z _start_methodZ_force_start_method� itertoolsr3 r �setr Z _close_stdinr Z_finalizer_registry�clearZ_run_after_forkers�infor5 Z_exit_function� SystemExitr1 r? rJ �sys�stderr�writer � tracebackr0 � print_excZ_flush_std_streams)r rN rO Z old_processrB �erY r r r � _bootstrap� sB zBaseProcess._bootstrap)N)r) � __module__� __qualname__�__doc__r r4 r5 r9 r: r* r> �propertyr0 �setterr r@ rB rD rC r7 rL r\ r r r r r > s* c @ s e Zd Zdd� ZdS )rA c C s, ddl m} |� d kr td��tt| �ffS )NrM )�get_spawning_popenzJPickling an AuthenticationString object is disallowed for security reasons)rO rb � TypeErrorrA �bytes)r rb r r r � __reduce__ s z AuthenticationString.__reduce__N)r) r] r^ re r r r r rA s rA c @ s e Zd Zdd� ZdS )� _MainProcessc C s2 f | _ d| _d | _d | _ttjd��dd�| _d S )NZ MainProcess� z/mp)r@ Z semprefix)r r+ r" r rA r �urandomr )r r r r r4 - s z_MainProcess.__init__N)r) r] r^ r4 r r r r rf + s rf rM � ZSIG�_) �__all__r rV �signalrQ � _weakrefsetr �path�abspath�getcwdZ ORIGINAL_DIR�OSErrorr r r �objectr rd rA rf r r3 r rR r rK r �__dict__�itemsr0 Zsignumr, r r r r �<module> s2 ` __pycache__/util.cpython-36.pyc 0000644 00000023110 15204243006 0012350 0 ustar 00 3 \n. � @ s� d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ ddlm Z ddddd d d d d ddddddgZ d Z dZ dZ dZdZdZdZdadadd� Zd d� Zd d� Zd d� Zd d � Zd6d d �Zd!d � Zej� Z ej � Z d"d#� Z d$d � Z i Z!ej � Z"G d%d� de#�Z$d7d&d'�Z%d(d � Z&da'eee%e j(e j)fd)d*�Z*ej+e*� G d+d� de#�Z,G d,d� dej-�Z.ye j/d-�Z0W n e1k �r� d.Z0Y nX d/d� Z2d0d1� Z3d2d3� Z4d4d5� Z5dS )8� N)� _args_from_interpreter_flags� )�process� sub_debug�debug�info� sub_warning� get_logger� log_to_stderr� get_temp_dir�register_after_fork� is_exiting�Finalize�ForkAwareThreadLock�ForkAwareLocal�close_all_fds_except�SUBDEBUG� SUBWARNING� � � � Zmultiprocessingz+[%(levelname)s/%(processName)s] %(message)sFc G s t rt jt| f|�� d S )N)�_logger�logr )�msg�args� r �,/usr/lib64/python3.6/multiprocessing/util.pyr , s c G s t rt jt| f|�� d S )N)r r �DEBUG)r r r r r r 0 s c G s t rt jt| f|�� d S )N)r r �INFO)r r r r r r 4 s c G s t rt jt| f|�� d S )N)r r r )r r r r r r 8 s c C s| ddl } | j� z\tsj| jt�adt_ttd�rFtjt � tj t � n$tj j t f i f� tj j t f i f� W d| j� X tS )z0 Returns logger used by multiprocessing r N� unregister)�loggingZ _acquireLockr Z getLogger� LOGGER_NAMEZ propagate�hasattr�atexitr �_exit_function�registerZ _exithandlers�remove�appendZ _releaseLock)r! r r r r < s c C sJ ddl }t� }|jt�}|j� }|j|� |j|� | rB|j| � dat S )zB Turn on logging and add a handler which prints to stderr r NT) r! r Z Formatter�DEFAULT_LOGGING_FORMATZ StreamHandlerZ setFormatterZ addHandlerZsetLevel�_log_to_stderrr )�levelr! ZloggerZ formatterZhandlerr r r r W s c C sd t j� jjd�} | d kr`dd l}dd l}|jdd�} td| � td |j | gdd� | t j� jd< | S ) N�tempdirr zpymp-)�prefixzcreated temp directory %s�d )r � exitpriorityi����) r �current_processZ_config�get�shutil�tempfileZmkdtempr r Zrmtree)r, r2 r3 r r r r m s c C sj t tj� �} | j� xP| D ]H\\}}}}y ||� W q tk r` } ztd|� W Y d d }~X qX qW d S )Nz after forker raised exception %s)�list�_afterfork_registry�items�sort� Exceptionr )r6 �indexZident�func�obj�er r r �_run_after_forkers s r= c C s | t tt�t| �|f< d S )N)r5 �next�_afterfork_counter�id)r; r: r r r r � s c @ sL e Zd ZdZf ddfdd�Zdeeejfdd�Z dd� Z d d � Z d d � Z dS ) r zA Class which supports object finalization using weakrefs Nc C sx |d kst |�tkst�|d k r0tj|| �| _n |d k s<t�|| _|| _|pNi | _|t t �f| _ t j � | _| t| j < d S )N)�type�int�AssertionError�weakref�ref�_weakref� _callback�_args�_kwargsr> �_finalizer_counter�_key�os�getpid�_pid�_finalizer_registry)�selfr; �callbackr �kwargsr/ r r r �__init__� s zFinalize.__init__c C s� y || j = W n tk r( |d� Y nbX | j|� krD|d� d}n$|d| j| j| j� | j| j| j�}d | _ | _ | _ | _| _ |S dS )zQ Run the callback unless it has already been called or cancelled z finalizer no longer registeredz+finalizer ignored because different processNz/finalizer calling %s with args %s and kwargs %s)rK �KeyErrorrN rG rH rI rF )rP �wrrO r rM �resr r r �__call__� s zFinalize.__call__c C sD y t | j= W n tk r Y n X d | _ | _ | _ | _| _dS )z3 Cancel finalization of the object N)rO rK rT rF rG rH rI )rP r r r �cancel� s zFinalize.cancelc C s | j tkS )zS Return whether this finalizer is still waiting to invoke callback )rK rO )rP r r r � still_active� s zFinalize.still_activec C s� y | j � }W n ttfk r( d }Y nX |d kr>d| jj S d| jjt| jd| j�f }| jrr|dt| j� 7 }| j r�|dt| j � 7 }| j d d k r�|dt| j d � 7 }|d S ) Nz<%s object, dead>z<%s object, callback=%s�__name__z, args=z , kwargs=r z, exitprority=�>) rF �AttributeError� TypeError� __class__rZ �getattrrG rH �strrI rK )rP r; �xr r r �__repr__� s zFinalize.__repr__) rZ � __module__� __qualname__�__doc__rS rO r rL rM rW rX rY rb r r r r r � s c s� t dkr dS �dkr dd� � n �fdd�� � fdd�tt �D �}|jdd� xX|D ]P}t j|�}|dk rRtd |� y |� W qR tk r� d dl}|j� Y qRX qRW �dkr�t j� dS ) z� Run all finalizers whose exit priority is not None and at least minpriority Finalizers with highest priority are called first; finalizers with the same priority will be called in reverse order of creation. Nc S s | d d k S )Nr r )�pr r r �<lambda>� s z!_run_finalizers.<locals>.<lambda>c s | d d k o| d � kS )Nr r )rf )� minpriorityr r rg � s c s g | ]}� |�r|�qS r r )�.0�key)�fr r � <listcomp>� s z#_run_finalizers.<locals>.<listcomp>T)�reversez calling %sr ) rO r4 r7 r1 r r8 � traceback� print_exc�clear)rh �keysrj � finalizerrn r )rk rh r �_run_finalizers� s$ rs c C s t p t dkS )z6 Returns true if the process is shutting down N)�_exitingr r r r r s c C s� t s�da | d� |d� |d� |� d k rzx*|� D ] }|jr2| d|j� |jj� q2W x"|� D ]}| d|j� |j� q^W |d� |� d S )NTzprocess shutting downz2running all "atexit" finalizers with priority >= 0r z!calling terminate() for daemon %sz calling join() for process %sz)running the remaining "atexit" finalizers)rt Zdaemon�nameZ_popenZ terminate�join)r r rs �active_childrenr0 rf r r r r% s r% c @ s, e Zd Zdd� Zdd� Zdd� Zdd� Zd S ) r c C s | j � t| tj � d S )N)�_resetr r )rP r r r rS K s z ForkAwareThreadLock.__init__c C s" t j� | _| jj| _| jj| _d S )N)� threadingZLock�_lock�acquire�release)rP r r r rx O s zForkAwareThreadLock._resetc C s | j j� S )N)rz � __enter__)rP r r r r} T s z ForkAwareThreadLock.__enter__c G s | j j|� S )N)rz �__exit__)rP r r r r r~ W s z ForkAwareThreadLock.__exit__N)rZ rc rd rS rx r} r~ r r r r r J s c @ s e Zd Zdd� Zdd� ZdS )r c C s t | dd� � d S )Nc S s | j j� S )N)�__dict__rp )r; r r r rg ] s z)ForkAwareLocal.__init__.<locals>.<lambda>)r )rP r r r rS \ s zForkAwareLocal.__init__c C s t | �f fS )N)rA )rP r r r � __reduce__^ s zForkAwareLocal.__reduce__N)rZ rc rd rS r� r r r r r [ s � SC_OPEN_MAX� c C sf t | �dtg } | j� | d tks,td��x4tt| �d �D ] }tj| | d | |d � q>W d S )Nr z fd too large���r� )r4 �MAXFDr7 rC �range�lenrL � closerange)Zfds�ir r r r j s c C s� t jd krd S yt jj� W n ttfk r4 Y nX y@tjtjtj�} yt| dd�t _W n tj| � � Y nX W n ttfk r� Y nX d S )NF)�closefd) �sys�stdin�close�OSError� ValueErrorrL �open�devnull�O_RDONLY)�fdr r r � _close_stdint s r� c C sT yt jj� W n ttfk r& Y nX yt jj� W n ttfk rN Y nX d S )N)r� �stdout�flushr\ r� �stderrr r r r �_flush_std_streams� s r� c C st dd l }tttt|���}tj� \}}z2|j|tj| �gd|d d ddddd d ||ddd �S tj |� tj |� X d S ) Nr Tr Fr� r� r� r� r� r� ) �_posixsubprocess�tuple�sorted�maprB rL �pipeZ fork_exec�fsencoder� )�pathr Zpassfdsr� Z errpipe_readZ errpipe_writer r r �spawnv_passfds� s r� )N)N)6rL � itertoolsr� rD r$ ry � subprocessr � r �__all__ZNOTSETr r r r r" r) r r* r r r r r r r �WeakValueDictionaryr5 �countr? r= r rO rJ �objectr rs r rt rw r0 r% r& r Zlocalr �sysconfr� r8 r r� r� r� r r r r �<module> sf S ,( __pycache__/connection.cpython-36.opt-1.pyc 0000644 00000057737 15204243006 0014517 0 ustar 00 3 �I�iVy � @ s d dddgZ ddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl m Z ddl m Z m Z dd lmZ ejZy$ddlZdd lmZmZmZmZ W n$ ek r� ejd kr dZY nX d Zd ZdZej � Z dZ dgZ e ed��rdZ e dg7 Z ejd k�r dZ e dg7 Z efdd�Z!dd� Z"dd� Z#dd� Z$dd� Z%G d d � d �Z&e�rnG d d � d e&�Z'G d d!� d!e&�Z(G d"d� de)�Z*dPd#d �Z+ejd k�r�dQd%d�Z,n dRd&d�Z,G d'd(� d(e)�Z-d)d*� Z.ejd k�r�G d+d,� d,e)�Z/d-d.� Z0d/Z1d0Z2d1Z3d2Z4d3d4� Z5d5d6� Z6G d7d8� d8e)�Z7d9d:� Z8d;d<� Z9G d=d>� d>e*�Z:d?d@� Z;ejd k�rzdAdB� Z<ej=ej>hZ?dSdCd�Z@n,ddlAZAe eAdD��r�eAjBZCneAjDZCdTdEd�Z@ejd k�r�dFdG� ZEdHdI� ZFejGe(eE� dJdK� ZHdLdM� ZIejGe'eH� n dNdG� ZEdOdI� ZFejGe(eE� dS )U�Client�Listener�Pipe�wait� N� )�util)�AuthenticationError�BufferTooShort)� reduction)� WAIT_OBJECT_0�WAIT_ABANDONED_0� WAIT_TIMEOUT�INFINITE�win32i g 4@Zsha256�AF_INET�AF_UNIX�AF_PIPEc C s t j� | S )N)�time� monotonic)�timeout� r �2/usr/lib64/python3.6/multiprocessing/connection.py� _init_timeout? s r c C s t j� | kS )N)r r )�tr r r �_check_timeoutB s r c C sX | dkr d S | dkr&t jdtj� d�S | dkrLt jdtj� tt�f d d�S td ��d S ) z? Return an arbitrary free address for the given family r � localhostr r z listener-)�prefix�dirr z\\.\pipe\pyc-%d-%d-� zunrecognized familyN)r r ) �tempfileZmktempr Z get_temp_dir�os�getpid�next� _mmap_counter� ValueError)�familyr r r �arbitrary_addressI s r& c C sJ t jdkr | dkr td| ��t jdkrF| dkrFtt| �sFtd| ��dS )zD Checks if the family is valid for the current environment. r r z Family %s is not recognized.r N)�sys�platformr$ �hasattr�socket)r% r r r �_validate_familyW s r+ c C sJ t | �tkrdS t | �tkr*| jd�r*dS t | �tkr:dS td| ��dS )z] Return the types of the address This can be 'AF_INET', 'AF_UNIX', or 'AF_PIPE' r z\\r r z address type of %r unrecognizedN)�type�tuple�str� startswithr$ )�addressr r r � address_typec s r1 c @ s� e Zd ZdZd+dd�Zdd� Zdd� Zd d � Zd d � Zd d� Z e dd� �Z e dd� �Z e dd� �Z dd� Zdd� Zd,dd�Zd d � Zd-d d �Zd.d d!�Zd"d#� Zd/d%d&�Zd'd(� Zd)d*� ZdS )0�_ConnectionBaseNTc C sB |j � }|dk rtd��| r,| r,td��|| _|| _|| _d S )Nr zinvalid handlez6at least one of `readable` and `writable` must be True)� __index__r$ �_handle� _readable� _writable)�self�handle�readable�writabler r r �__init__y s z_ConnectionBase.__init__c C s | j d k r| j� d S )N)r4 �_close)r7 r r r �__del__� s z_ConnectionBase.__del__c C s | j d krtd��d S )Nzhandle is closed)r4 �OSError)r7 r r r � _check_closed� s z _ConnectionBase._check_closedc C s | j std��d S )Nzconnection is write-only)r5 r> )r7 r r r �_check_readable� s z _ConnectionBase._check_readablec C s | j std��d S )Nzconnection is read-only)r6 r> )r7 r r r �_check_writable� s z _ConnectionBase._check_writablec C s" | j rd| _n| j� td��d S )NFzbad message length)r6 r5 �closer> )r7 r r r �_bad_message_length� s z#_ConnectionBase._bad_message_lengthc C s | j dkS )z True if the connection is closedN)r4 )r7 r r r �closed� s z_ConnectionBase.closedc C s | j S )z"True if the connection is readable)r5 )r7 r r r r9 � s z_ConnectionBase.readablec C s | j S )z"True if the connection is writable)r6 )r7 r r r r: � s z_ConnectionBase.writablec C s | j � | jS )z+File descriptor or handle of the connection)r? r4 )r7 r r r �fileno� s z_ConnectionBase.filenoc C s$ | j dk r z | j� W dd| _ X dS )zClose the connectionN)r4 r< )r7 r r r rB � s z_ConnectionBase.closer c C s� | j � | j� t|�}|jdkr.tt|��}t|�}|dk rFtd��||k rVtd��|dkrh|| }n&|dk rztd��n|| |kr�td��| j|||| � � dS )z,Send the bytes data from a bytes-like objectr r zoffset is negativezbuffer length < offsetNzsize is negativez buffer length < offset + size)r? rA � memoryview�itemsize�bytes�lenr$ � _send_bytes)r7 �buf�offset�size�m�nr r r � send_bytes� s" z_ConnectionBase.send_bytesc C s$ | j � | j� | jtj|�� dS )zSend a (picklable) objectN)r? rA rJ �_ForkingPickler�dumps)r7 �objr r r �send� s z_ConnectionBase.sendc C sJ | j � | j� |dk r(|dk r(td��| j|�}|dkrB| j� |j� S )z7 Receive bytes data as a bytes object. Nr znegative maxlength)r? r@ r$ � _recv_bytesrC �getvalue)r7 Z maxlengthrK r r r � recv_bytes� s z_ConnectionBase.recv_bytesc C s� | j � | j� t|���}|j}|t|� }|dk r>td��n||krNtd��| j� }|j� }||| k rvt|j � ��|j d� |j ||| || | � � |S Q R X dS )zq Receive bytes data into a writeable bytes-like object. Return the number of bytes read. r znegative offsetzoffset too largeN) r? r@ rF rG rI r$ rU �tellr rV �seek�readinto)r7 rK rL rN rG Zbytesize�resultrM r r r �recv_bytes_into� s" z _ConnectionBase.recv_bytes_intoc C s& | j � | j� | j� }tj|j� �S )z Receive a (picklable) object)r? r@ rU rQ �loads� getbuffer)r7 rK r r r �recv� s z_ConnectionBase.recv� c C s | j � | j� | j|�S )z/Whether there is any input available to be read)r? r@ �_poll)r7 r r r r �poll s z_ConnectionBase.pollc C s | S )Nr )r7 r r r � __enter__ s z_ConnectionBase.__enter__c C s | j � d S )N)rB )r7 �exc_type� exc_value�exc_tbr r r �__exit__ s z_ConnectionBase.__exit__)TT)r N)N)r )r` )�__name__� __module__� __qualname__r4 r; r= r? r@ rA rC �propertyrD r9 r: rE rB rP rT rW r\ r_ rb rc rg r r r r r2 v s( r2 c @ sD e Zd ZdZdZejfdd�Zdd� Zddd �Z d d � Z d d � Z dS )�PipeConnectionz� Connection class based on a Windows named pipe. Overlapped I/O is used, so the handles must have been created with FILE_FLAG_OVERLAPPED. Fc C s || j � d S )N)r4 )r7 Z _CloseHandler r r r< s zPipeConnection._closec C sh t j| j|dd�\}}z<y |t jkr6t j|jgdt�}W n |j� � Y nX W d |jd�\}}X d S )NT)� overlappedF) �_winapiZ WriteFiler4 �ERROR_IO_PENDING�WaitForMultipleObjects�eventr �cancel�GetOverlappedResult)r7 rK �ov�err�waitresZnwrittenr r r rJ s zPipeConnection._send_bytesNc C s | j rd| _ tj� S |d kr dnt|d�}y�tj| j|dd�\}}z<y |tjkrbtj|j gdt �}W n |j � � Y nX W d |j d�\}}|dkr�tj� }|j |j� � |S |tjkr�| j||�S X W n8 tk �r } z|jtjkr�t�n� W Y d d }~X nX td��d S )NF� T)rm r z.shouldn't get here; expected KeyboardInterrupt)�_got_empty_message�io�BytesIO�minrn �ReadFiler4 ro rp rq r rr rs �writer^ ZERROR_MORE_DATA�_get_more_datar> �winerror�ERROR_BROKEN_PIPE�EOFError� RuntimeError) r7 �maxsizeZbsizert ru rv Znread�f�er r r rU * s6 zPipeConnection._recv_bytesc C s. | j stj| j�d dkr dS tt| g|��S )Nr T)rx rn � PeekNamedPiper4 �boolr )r7 r r r r ra J s zPipeConnection._pollc C s� |j � }tj� }|j|� tj| j�d }|d k rJt|�| |krJ| j� tj | j|dd�\}}|j d�\}}|j|j � � |S )Nr T)rm ) r^ ry rz r} rn r� r4 rI rC r| rs )r7 rt r� rK r� �leftru Zrbytesr r r r~ P s z PipeConnection._get_more_data)N) rh ri rj �__doc__rx rn � CloseHandler< rJ rU ra r~ r r r r rl s rl c @ s| e Zd ZdZer,ejfdd�ZejZ ej Z ne j fdd�Ze jZ e jZ e fdd�Ze fdd�Zd d � Zdd d �Zdd� Zd S )� Connectionzo Connection class based on an arbitrary file descriptor (Unix only), or a socket handle (Windows). c C s || j � d S )N)r4 )r7 r< r r r r< g s zConnection._closec C s || j � d S )N)r4 )r7 r< r r r r< l s c C s< t |�}x.|| j|�}||8 }|dkr(P ||d � }q W d S )Nr )rI r4 )r7 rK r} � remainingrO r r r �_sendq s zConnection._sendc C sf t j� }| j}|}xN|dkr`|||�}t|�}|dkrL||krDt�ntd��|j|� ||8 }qW |S )Nr z got end of file during message)ry rz r4 rI r� r> r} )r7 rM �readrK r8 r� �chunkrO r r r �_recvz s zConnection._recvc C sD t |�}tjd|�}|dkr2| j|� | j|� n| j|| � d S )Nz!ii @ )rI �structZpackr� )r7 rK rO �headerr r r rJ � s zConnection._send_bytesNc C s: | j d�}tjd|j� �\}|d k r0||kr0d S | j |�S )N� z!i)r� r� ZunpackrV )r7 r� rK rM r r r rU � s zConnection._recv_bytesc C s t | g|�}t|�S )N)r r� )r7 r �rr r r ra � s zConnection._poll)N)rh ri rj r� rn �_multiprocessingZ closesocketr< rT Z_writer_ Z_readr rB r} r� r� r� rJ rU ra r r r r r� ` s r� c @ sR e Zd ZdZddd�Zdd� Zdd � Zed d � �Zed d � �Z d d� Z dd� Z dS )r z� Returns a listener object. This is a wrapper for a bound socket which is 'listening' for connections, or for a Windows named pipe. Nr c C sr |p|rt |�pt}|p t|�}t|� |dkr>t||�| _nt|||�| _|d k rht|t� rht d��|| _ d S )Nr z authkey should be a byte string) r1 �default_familyr& r+ � PipeListener� _listener�SocketListener� isinstancerH � TypeError�_authkey)r7 r0 r% �backlog�authkeyr r r r; � s zListener.__init__c C s> | j dkrtd��| j j� }| jr:t|| j� t|| j� |S )zz Accept a connection on the bound socket or named pipe of `self`. Returns a `Connection` object. Nzlistener is closed)r� r> �acceptr� �deliver_challenge�answer_challenge)r7 �cr r r r� � s zListener.acceptc C s | j }|dk r d| _ |j� dS )zA Close the bound socket or named pipe of `self`. N)r� rB )r7 Zlistenerr r r rB � s zListener.closec C s | j jS )N)r� �_address)r7 r r r �<lambda>� s zListener.<lambda>c C s | j jS )N)r� �_last_accepted)r7 r r r r� � s c C s | S )Nr )r7 r r r rc � s zListener.__enter__c C s | j � d S )N)rB )r7 rd re rf r r r rg � s zListener.__exit__)NNr N) rh ri rj r� r; r� rB rk r0 Z last_acceptedrc rg r r r r r � s c C sj |p t | �}t|� |dkr&t| �}nt| �}|dk rJt|t� rJtd��|dk rft||� t||� |S )z= Returns a connection to the address of a `Listener` r Nz authkey should be a byte string) r1 r+ � PipeClient� SocketClientr� rH r� r� r� )r0 r% r� r� r r r r � s Tc C sj | r>t j� \}}|jd� |jd� t|j� �}t|j� �}n$tj� \}}t|dd�}t|dd�}||fS )zL Returns pair of connection objects at either end of a pipe TF)r: )r9 )r* Z socketpair� setblockingr� �detachr �pipe)�duplex�s1�s2�c1�c2Zfd1Zfd2r r r r � s c C s� t d�}| r*tj}tjtjB }tt }}ntj}tj}dt }}tj||tjB tj B tj tj B tj B d||tj tj�}tj||dtjtjtjtj�}tj|tj dd� tj|dd�}|jd�\} } t|| d�} t|| d�} | | fS ) zL Returns pair of connection objects at either end of a pipe r r r NT)rm )r: )r9 )r& rn �PIPE_ACCESS_DUPLEX� GENERIC_READ� GENERIC_WRITE�BUFSIZEZPIPE_ACCESS_INBOUND�CreateNamedPipe�FILE_FLAG_OVERLAPPED� FILE_FLAG_FIRST_PIPE_INSTANCE�PIPE_TYPE_MESSAGE�PIPE_READMODE_MESSAGE� PIPE_WAIT�NMPWAIT_WAIT_FOREVER�NULL� CreateFile� OPEN_EXISTING�SetNamedPipeHandleState�ConnectNamedPipers rl ) r� r0 Zopenmode�accessZobsizeZibsizeZh1Zh2rm �_ru r� r� r r r r s2 c @ s* e Zd ZdZd dd�Zdd� Zdd� Zd S ) r� zO Representation of a socket which is bound to an address and listening r c C s� t j tt |��| _yRtjdkr2| jjt jt jd� | jjd� | jj |� | jj |� | jj � | _ W n t k r� | jj� � Y nX || _d | _|dkr�tj| tj|fdd�| _nd | _d S )N�posixr Tr r )�args� exitpriority)r* �getattr�_socketr �nameZ setsockoptZ SOL_SOCKETZ SO_REUSEADDRr� ZbindZlistenZ getsocknamer� r> rB Z_familyr� r �Finalize�unlink�_unlink)r7 r0 r% r� r r r r; < s$ zSocketListener.__init__c C s& | j j� \}| _|jd� t|j� �S )NT)r� r� r� r� r� r� )r7 �sr r r r� T s zSocketListener.acceptc C s0 z| j j� W d | j}|d k r*d | _|� X d S )N)r� rB r� )r7 r� r r r rB Y s zSocketListener.closeN)r )rh ri rj r� r; r� rB r r r r r� 8 s r� c C sD t | �}tjtt|���"}|jd� |j| � t|j� �S Q R X dS )zO Return a connection object connected to the socket given by `address` TN)r1 r* r� r� Zconnectr� r� )r0 r% r� r r r r� c s r� c @ s8 e Zd ZdZd dd�Zd dd�Zdd � Zed d � �ZdS )r� z0 Representation of a named pipe Nc C sL || _ | jdd�g| _d | _tjd| j � tj| tj| j| j fdd�| _ d S )NT)�firstz listener created with address=%rr )r� r� ) r� � _new_handle� _handle_queuer� r � sub_debugr� r� �_finalize_pipe_listenerrB )r7 r0 r� r r r r; w s zPipeListener.__init__Fc C sH t jt jB }|r|t jO }t j| j|t jt jB t jB t j t t t j t j �S )N) rn r� r� r� r� r� r� r� r� ZPIPE_UNLIMITED_INSTANCESr� r� r� )r7 r� �flagsr r r r� � s zPipeListener._new_handlec C s� | j j| j� � | j jd�}ytj|dd�}W n0 tk r^ } z|jtjkrN� W Y d d }~X nPX z<ytj |j gdt �}W n |j � tj |� � Y nX W d |jd�\}}X t|�S )Nr T)rm F)r� �appendr� �poprn r� r> r Z ERROR_NO_DATArp rq r rr r� rs rl )r7 r8 rt r� �resr� ru r r r r� � s" zPipeListener.acceptc C s( t jd|� x| D ]}tj|� qW d S )Nz closing listener with address=%r)r r� rn r� )Zqueuer0 r8 r r r r� � s z$PipeListener._finalize_pipe_listener)N)F) rh ri rj r� r; r� r� � staticmethodr� r r r r r� s s r� c C s� t � }x~y6tj| d� tj| tjtjB dtjtjtjtj�}W n> t k r| } z"|j tj tj fksjt |�rl� W Y dd}~X qX P qW � tj|tjdd� t|�S )zU Return a connection object connected to the pipe given by `address` i� r N)r rn Z WaitNamedPiper� r� r� r� r� r� r> r ZERROR_SEM_TIMEOUTZERROR_PIPE_BUSYr r� r� rl )r0 r �hr� r r r r� � s" r� � s #CHALLENGE#s #WELCOME#s #FAILURE#c C sf dd l }tjt�}| jt| � |j||t�j� }| j d�}||krP| jt � n| jt � t d��d S )Nr � zdigest received was wrong) �hmacr �urandom�MESSAGE_LENGTHrP � CHALLENGE�new�HMAC_DIGEST_NAME�digestrW �WELCOME�FAILUREr )� connectionr� r� �messager� �responser r r r� � s r� c C s\ dd l }| jd�}|tt�d � }|j||t�j� }| j|� | jd�}|tkrXt d��d S )Nr r� zdigest sent was rejected) r� rW rI r� r� r� r� rP r� r )r� r� r� r� r� r� r r r r� � s r� c @ s$ e Zd Zdd� Zdd� Zdd� ZdS )�ConnectionWrapperc C s: || _ || _|| _x"dD ]}t||�}t| ||� qW d S )NrE rB rb rW rP )rE rB rb rW rP )�_conn�_dumps�_loadsr� �setattr)r7 �connrR r] �attrrS r r r r; � s zConnectionWrapper.__init__c C s | j |�}| jj|� d S )N)r� r� rP )r7 rS r� r r r rT � s zConnectionWrapper.sendc C s | j j� }| j|�S )N)r� rW r� )r7 r� r r r r_ � s zConnectionWrapper.recvN)rh ri rj r; rT r_ r r r r r� � s r� c C s t j| fd d d d�jd�S )Nr zutf-8)� xmlrpclibrR �encode)rS r r r � _xml_dumps� s r� c C s t j| jd��\\}}|S )Nzutf-8)r� r] �decode)r� rS �methodr r r � _xml_loads� s r� c @ s e Zd Zdd� ZdS )� XmlListenerc C s dd l jatj| �}t|tt�S )Nr )� xmlrpc.client�clientr� r r� r� r� r� )r7 rS r r r r� s zXmlListener.acceptN)rh ri rj r� r r r r r� s r� c O s dd l jatt| |�tt�S )Nr )r� r� r� r� r r� r� )r� �kwdsr r r � XmlClient s r� c C s� t | �}g }x�|r�tj|d|�}|tkr,P n\t| koFtt|� k n rV|t8 }n2t| koptt|� k n r�|t8 }ntd��|j|| � ||d d � }d}qW |S )NFzShould not get herer r ) �listrn rp r r rI r r� r� )Zhandlesr �L�readyr� r r r �_exhaustive_wait s r� c . sn |dkrt }n |dk r d}nt|d d �}t| �} i �g }t� � t� }�zH�x2| D �](}yt|d�}W n tk r� |�|j� < Y qTX ytj|� dd�\}}W n8 t k r� } z d|j }}|t krʂ W Y dd}~X nX |tj kr�|j |� |�|j< qT|�rptj� dd� dk�rpy|jd �\} }W n* t k �rT } z |j }W Y dd}~X nX | �rpt|d ��rpd|_� j|� d}qTW t�j� |�}W dx|D ]}|j� �q�W x�|D ]�}y|jd�\} }W n6 t k �r� } z|j }|t k�r� W Y dd}~X nX |tjk�r��|j }� j|� |dk�r�t|d ��r�d|_�q�W X � j�fd d �|D �� � fd d�| D �S )z� Wait till an object in object_list is ready/readable. Returns list of those objects in object_list which are ready/readable. Nr i� g �?rE T� � Frx c 3 s | ]}� | V qd S )Nr )�.0r� )�waithandle_to_objr r � <genexpr>v s zwait.<locals>.<genexpr>c s g | ]}|� kr|�qS r r )r� �o)� ready_objectsr r � <listcomp>w s zwait.<locals>.<listcomp>)r� r� )r �intr� �setr� �AttributeErrorr3 rn r| r> r � _ready_errorsro r� rq r' Zgetwindowsversionrs r) rx �addr� �keysrr ZERROR_OPERATION_ABORTED�update) � object_listr Zov_listZ ready_handlesr rE rt ru r� r� r )r r� r r + sh � PollSelectorc C s� t � �x}x| D ]}|j|tj� qW |dk r8tj� | }x@|j|�}|rVdd� |D �S |dk r:|tj� }|dk r:|S q:W W dQ R X dS )z� Wait till an object in object_list is ready/readable. Returns list of those objects in object_list which are ready/readable. Nc S s g | ]\}}|j �qS r )Zfileobj)r� �keyZeventsr r r r � s zwait.<locals>.<listcomp>r )� _WaitSelector�register� selectorsZ EVENT_READr r Zselect)r r ZselectorrS Zdeadliner� r r r r � s c C sN | j � }tj|tjtj��*}ddlm} |j|�}t|| j | j ffS Q R X d S )Nr )�resource_sharer) rE r* Zfromfdr Z SOCK_STREAMr r Z DupSocket�rebuild_connectionr9 r: )r� r8 r� r �dsr r r �reduce_connection� s r c C s | j � }t|j � ||�S )N)r� r� )r r9 r: Zsockr r r r � s r c C sB | j r tjnd| jrtjndB }tj| j� |�}t|| j | jffS )Nr ) r9 rn ZFILE_GENERIC_READr: ZFILE_GENERIC_WRITEr Z DupHandlerE �rebuild_pipe_connection)r� r� �dhr r r �reduce_pipe_connection� s r c C s | j � }t|||�S )N)r� rl )r r9 r: r8 r r r r � s r c C s t j| j� �}t|| j| jffS )N)r ZDupFdrE r r9 r: )r� �dfr r r r � s c C s | j � }t|||�S )N)r� r� )r r9 r: �fdr r r r � s )NN)T)T)N)N)J�__all__ry r r' r* r� r r � itertoolsr� r r r r �contextr ZForkingPicklerrQ rn r r r r � ImportErrorr( r� ZCONNECTION_TIMEOUTr� �countr# r� Zfamiliesr) r r r& r+ r1 r2 rl r� �objectr r r r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� ZERROR_NETNAME_DELETEDr r r r r ZSelectSelectorr r r r r r r r r �<module> s� PJ8 ,+ 8 P __pycache__/heap.cpython-36.opt-2.pyc 0000644 00000013405 15204243007 0013257 0 ustar 00 3 \ � @ s� d dl Z d dlZd dlZd dlZd dlZd dlZddlmZmZ ddl m Z dgZ ej dkrvd dl Z G dd� de�Zn,G d d� de�Zd d � Zd d � Zejee� G dd� de�ZG dd� de�ZdS )� N� )� reduction�assert_spawning)�util� BufferWrapperZwin32c @ s, e Zd Zej� Zdd� Zdd� Zdd� ZdS )�Arenac C sz || _ xTtd�D ]@}dtj� t| j�f }tjd||d�}tj� dkrHP |j � qW t d��|| _ || _ | j | j f| _ d S )N�d z pym-%d-%sr )�tagnamer z Cannot find name for new mmap���)�size�range�os�getpid�next�_rand�mmap�_winapiZ GetLastError�close�FileExistsError�name�buffer�_state)�selfr �ir Zbuf� r �,/usr/lib64/python3.6/multiprocessing/heap.py�__init__" s zArena.__init__c C s t | � | jS )N)r r )r r r r � __getstate__1 s zArena.__getstate__c C s, | \| _ | _| _tjd| j | jd�| _d S )Nr )r r )r r r r r )r �stater r r � __setstate__5 s zArena.__setstate__N) �__name__� __module__� __qualname__�tempfileZ_RandomNameSequencer r r r r r r r r s r c @ s e Zd Zddd�ZdS )r r c C s� || _ || _|d kr�tjdtj� tj� d�\| _}tj|� tj | tj | jf� t | jddd��N}d }||kr�d| }x t || �D ]}|j |� q�W ~|j d|| � W d Q R X tj| j| j �| _d S ) Nr zpym-%d-)�prefix�dir�wbF)�closefdi � r i )r �fdr# Zmkstempr r r Z get_temp_dir�unlink�Finalizer �openr �writer r )r r r) r �fZbsZzeros�_r r r r @ s" zArena.__init__Nr )r )r r! r" r r r r r r > s c C s( | j dkrtd��t| jtj| j �ffS )Nr zDArena is unpicklable because forking was enabled when it was createdr )r) � ValueError� rebuild_arenar r ZDupFd)�ar r r � reduce_arenaS s r3 c C s t | |j� �S )N)r �detach)r Zdupfdr r r r1 Y s r1 c @ sZ e Zd ZdZejfdd�Zedd� �Zdd� Z dd � Z d d � Z d d � Z dd� Z dd� ZdS )�Heap� c C sJ t j� | _tj� | _|| _g | _i | _i | _ i | _ t � | _ g | _ g | _d S )N)r r �_lastpid� threadingZLock�_lock�_size�_lengths� _len_to_seq�_start_to_block�_stop_to_block�set�_allocated_blocks�_arenas�_pending_free_blocks)r r r r r r f s z Heap.__init__c C s |d }| | | @ S )Nr r )�nZ alignment�maskr r r �_roundups s z Heap._roundupc C s� t j| j|�}|t| j�krj| jt| j|�tj�}| jd9 _t j d|� t |�}| j j |� |d|fS | j| }| j| }|j� }|s�| j|= | j|= |\}}}| j||f= | j||f= |S )N� z"allocating a new mmap of length %dr )�bisectZ bisect_leftr; �lenrE �maxr: r �PAGESIZEr �infor rA �appendr< �popr= r> ) r r r �length�arena�seq�block�start�stopr r r �_mallocy s" z Heap._mallocc C s� |\}}}y| j ||f }W n tk r0 Y nX | j|�\}}y| j||f }W n tk rf Y nX | j|�\}}|||f}|| }y| j| j|� W n. tk r� |g| j|< tj| j|� Y nX || j||f< || j ||f< d S )N) r> �KeyError�_absorbr= r< rL rG Zinsortr; ) r rQ rO rR rS Z prev_blockr/ Z next_blockrN r r r �_free� s( z Heap._freec C s^ |\}}}| j ||f= | j||f= || }| j| }|j|� |sV| j|= | jj|� ||fS )N)r= r> r< �remover; )r rQ rO rR rS rN rP r r r rV � s z Heap._absorbc C sF x@y| j j� }W n tk r& P Y nX | jj|� | j|� qW d S )N)rB rM � IndexErrorr@ rX rW )r rQ r r r �_free_pending_blocks� s zHeap._free_pending_blocksc C sN | j jd�s| jj|� n0z"| j� | jj|� | j|� W d | j j� X d S )NF) r9 �acquirerB rL rZ r@ rX rW �release)r rQ r r r �free� s z Heap.freec C s� t j� | jkr| j� | j�h | j� | jt|d�| j�}| j |�\}}}|| }||k rj| j |||f� |||f}| j j |� |S Q R X d S )Nr ) r r r7 r r9 rZ rE rI � _alignmentrT rW r@ �add)r r rO rR rS Znew_stoprQ r r r �malloc� s z Heap.mallocN)r r! r" r^ r rJ r � staticmethodrE rT rW rV rZ r] r` r r r r r5 b s r5 c @ s" e Zd Ze� Zdd� Zdd� ZdS )r c C s0 t jj|�}||f| _tj| t jj|fd� d S )N)�args)r �_heapr` r r r+ r] )r r rQ r r r r � s zBufferWrapper.__init__c C s& | j \\}}}}t|j�||| � S )N)r � memoryviewr )r rO rR rS r r r r �create_memoryview� s z BufferWrapper.create_memoryviewN)r r! r" r5 rc r re r r r r r � s )rG r r �sysr# r8 �contextr r � r �__all__�platformr �objectr r3 r1 �registerr5 r r r r r �<module> s$ __pycache__/popen_spawn_win32.cpython-36.opt-1.pyc 0000644 00000005243 15204243007 0015715 0 ustar 00 3 \� � @ s� d dl Z d dlZd dlZd dlZd dlZddlmZmZmZ ddl m Z ddl m Z dgZ dZ ejdkoreed d �Zejj� jd �ZG d d� de�ZdS ) � N� )� reduction�get_spawning_popen�set_spawning_popen)�spawn)�util�Popeni Zwin32�frozenFzpythonservice.exec @ s> e Zd ZdZdZdd� Zdd� Zddd �Zd d � Zd d � Z dS )r z@ Start a subprocess to run the code of a process object r c C s t j|j�}tjd d�\}}tj|d�}t jtj � |d�}dj dd� |D ��}t |ddd���}y4tj t j � |d d d dd d d � \}} } } tj| � W n tj|� � Y nX | | _d | _|| _t|�| _tj| tj| jf� t| � z tj||� tj||� W d td � X W d Q R X d S ) Nr )Z parent_pidZ pipe_handle� c s s | ]}d | V qdS )z"%s"N� )�.0�xr r �9/usr/lib64/python3.6/multiprocessing/popen_spawn_win32.py� <genexpr>) s z!Popen.__init__.<locals>.<genexpr>�wbT)�closefdF)r Zget_preparation_data�_name�_winapiZ CreatePipe�msvcrtZopen_osfhandleZget_command_line�os�getpid�join�openZ CreateProcessZget_executableZ CloseHandle�pid� returncode�_handle�int�sentinelr ZFinalizer r �dump) �selfZ process_objZ prep_dataZrhandleZwhandleZwfd�cmdZto_childZhpZhtr �tidr r r �__init__ s2 zPopen.__init__c C s t j|| j�S )N)r Z duplicater )r Zhandler r r �duplicate_for_childE s zPopen.duplicate_for_childNc C st | j d krn|d krtj}ntdt|d d ��}tjt| j�|�}|tjkrntj| j�}|t krht j }|| _ | j S )Nr i� g �?) r r ZINFINITE�maxr ZWaitForSingleObjectr Z WAIT_OBJECT_0ZGetExitCodeProcess� TERMINATE�signal�SIGTERM)r �timeoutZmsecs�res�coder r r �waitI s z Popen.waitc C s | j dd�S )Nr )r( )r+ )r r r r �pollY s z Popen.pollc C sL | j d krHytjt| j�t� W n& tk rF | jdd�d krB� Y nX d S )Ng �?)r( )r r ZTerminateProcessr r r% �OSErrorr+ )r r r r � terminate\ s zPopen.terminate)N) �__name__� __module__� __qualname__�__doc__�methodr" r# r+ r, r. r r r r r s % )r r r& �sysr �contextr r r � r r �__all__r% �platform�getattrZWINEXE� executable�lower�endswithZ WINSERVICE�objectr r r r r �<module> s __pycache__/reduction.cpython-36.pyc 0000644 00000017427 15204243007 0013406 0 ustar 00 3 \ $ � @ s d dl mZmZ d dlZd dlZd dlZd dlZd dlZd dlZd dl Z ddl m Z ddddd gZ e j d kp�eed �o�eed �o�eejd �ZG dd� dej�ZejZd5dd �Ze j d k�re dddg7 Z d dlZd6dd�Zdd� Zdd� Zdd� ZG dd� de�ZnHe dddg7 Z d dlZe j d kZ d d� Z d d� Z d d� Zd d� Zd!d� Z d"d#� Z G d$d%� d%�Z!ee"e!� j#�e � d&d'� Z$ee"e%j&�e$� ee"e'j(�e$� d(d)� Z)d*d+� Z*eej+e)� e j d k�r�d,d-� Z,d.d/� Z-eeje,� n d0d-� Z,d1d/� Z-eeje,� G d2d3� d3ed4�Z.dS )7� )�ABCMeta�abstractmethodN� )�context� send_handle� recv_handle�ForkingPickler�register�dump�win32ZCMSG_LEN� SCM_RIGHTS�sendmsgc sJ e Zd ZdZi ZejZ� fdd�Ze dd� �Z e d dd��Z e j Z � ZS ) r z)Pickler subclass used by multiprocessing.c s* t � j|� | jj� | _| jj| j� d S )N)�super�__init__�_copyreg_dispatch_table�copy�dispatch_table�update�_extra_reducers)�self�args)� __class__� �1/usr/lib64/python3.6/multiprocessing/reduction.pyr & s zForkingPickler.__init__c C s || j |< dS )z&Register a reduce function for a type.N)r )�cls�type�reducer r r r + s zForkingPickler.registerNc C s t j� }| ||�j|� |j� S )N)�io�BytesIOr � getbuffer)r �obj�protocolZbufr r r �dumps0 s zForkingPickler.dumps)N)�__name__� __module__� __qualname__�__doc__r �copyregr r r � classmethodr r" �pickle�loads� __classcell__r r )r r r ! s c C s t ||�j| � dS )z3Replacement for pickle.dump() using ForkingPickler.N)r r )r �filer! r r r r : s � DupHandle� duplicate� steal_handleFc C s* |dkrt j� }t jt j� | |d|t j�S )z<Duplicate a handle. (target_process is a handle not a pid!)Nr )�_winapi�GetCurrentProcess�DuplicateHandle�DUPLICATE_SAME_ACCESS)�handleZtarget_processZ inheritabler r r r. G s c C sB t jt jd| �}z t j||t j� ddt jt jB �S t j|� X dS )z5Steal a handle from process identified by source_pid.Fr N)r0 � OpenProcess�PROCESS_DUP_HANDLEr2 r1 r3 �DUPLICATE_CLOSE_SOURCE� CloseHandle)Z source_pidr4 Zsource_process_handler r r r/ O s c C s t |tj|�}| j|� dS )z&Send a handle over a local connection.N)r- r0 r3 �send)�connr4 �destination_pidZdhr r r r [ s c C s | j � j� S )z)Receive a handle over a local connection.)�recv�detach)r: r r r r ` s c @ s" e Zd ZdZddd�Zdd� ZdS )r- z Picklable wrapper for a handle.Nc C s\ |d krt j� }tjtjd|�}z tjtj� |||dd�| _W d tj|� X || _ || _ d S )NFr ) �os�getpidr0 r5 r6 r2 r1 �_handler8 �_access�_pid)r r4 �access�pid�procr r r r f s zDupHandle.__init__c C sV | j tj� kr| jS tjtjd| j �}z tj|| jtj� | j dtj �S tj |� X dS )z1Get the handle. This should only be called once.FN) rB r> r? r@ r0 r5 r6 r2 r1 rA r7 r8 )r rE r r r r= u s zDupHandle.detach)N)r# r$ r% r& r r= r r r r r- d s �DupFd�sendfds�recvfds�darwinc C sV t j d|�}tt|�d g�}| j|gtjtj|fg� trR| jd�dkrRt d��dS )z,Send an array of fds over an AF_UNIX socket.�i� r � Az%did not receive acknowledgement of fdN) �array�bytes�lenr �socket� SOL_SOCKETr � ACKNOWLEDGEr< � RuntimeError)�sockZfds�msgr r r rG � s c C s� t j d�}|j| }| jdtj|��\}}}}| r>| r>t�y�trN| jd� t|�dkrjt dt|� ��|d \}} } |tj kr�| tj kr�t| �|j dkr�t �|j | � t|�d |d ks�t�t|�S W n t tfk r� Y nX t d��dS ) z/Receive an array of fds over an AF_UNIX socket.rJ r rL z received %d items of ancdatar rK zInvalid data receivedN)rM �itemsizeZrecvmsgrP Z CMSG_SPACE�EOFErrorrR r9 rO rS rQ r � ValueErrorZ frombytes�AssertionError�list� IndexError) rT �size�aZ bytes_sizerU Zancdata�flagsZaddrZ cmsg_levelZ cmsg_typeZ cmsg_datar r r rH � s, c C s2 t j| j� t jt j��}t||g� W dQ R X dS )z&Send a handle over a local connection.N)rP �fromfd�fileno�AF_UNIX� SOCK_STREAMrG )r: r4 r; �sr r r r � s c C s0 t j| j� t jt j��}t|d�d S Q R X dS )z)Receive a handle over a local connection.r r N)rP r_ r` ra rb rH )r: rc r r r r � s c C sF t j� }|dk r |j|j| ��S tr:ddlm} |j| �S td��dS )zReturn a wrapper for an fd.Nr )�resource_sharerz&SCM_RIGHTS appears not to be available)r Zget_spawning_popenrF Zduplicate_for_child�HAVE_SEND_HANDLE� rd rX )�fdZ popen_objrd r r r rF � s c C s2 | j d kr t| j| jjffS t| j | jjffS d S )N)�__self__�getattrr �__func__r# )�mr r r �_reduce_method� s rl c @ s e Zd Zdd� ZdS )�_Cc C s d S )Nr )r r r r �f� s z_C.fN)r# r$ r% rn r r r r rm � s rm c C s t | j| jffS )N)ri � __objclass__r# )rk r r r �_reduce_method_descriptor� s rp c C s t | j| j| jpi ffS )N)�_rebuild_partial�funcr �keywords)�pr r r �_reduce_partial� s ru c C s t j| f|�|�S )N)� functools�partial)rr r rs r r r rq � s rq c C s ddl m} t|| �ffS )Nr )� DupSocket)rd rx �_rebuild_socket)rc rx r r r �_reduce_socket� s rz c C s | j � S )N)r= )Zdsr r r ry � s ry c C s" t | j� �}t|| j| j| jffS )N)rF r` ry �familyr �proto)rc �dfr r r rz � s c C s | j � }tj||||d�S )N)r` )r= rP )r} r{ r r| rg r r r ry � s c @ sd e Zd ZdZeZeZeZeZeZe j dkr8e Z e Z e Z n eZeZeZeZeZeZeZeZdd� ZdS )�AbstractReducerz�Abstract base class for use in implementing a Reduction class suitable for use in replacing the standard reduction mechanism used in multiprocessing.r c G sN t tt� j�t� t ttj�t� t ttj �t� t t j t � t t j t� d S )N)r r rm rn rl rZ �appendrp �int�__add__rv rw ru rP rz )r r r r r r s zAbstractReducer.__init__N)r# r$ r% r& r r r r r �sys�platformr/ r. r- rG rH rF rl rp rq rz ry r r r r r r~ � s& r~ )� metaclass)N)NF)/�abcr r r' rv r r> r) rP r� rf r �__all__r� �hasattrre ZPicklerr r r r0 r. r/ r r �objectr- rM rR rG rH rF rl rm r rn rp rZ r r� r� ru rq rw rz ry r~ r r r r �<module> sb # __pycache__/popen_fork.cpython-36.pyc 0000644 00000004124 15204243007 0013542 0 ustar 00 3 \ � @ s> d dl Z d dlZd dlZddlmZ dgZG dd� de�ZdS )� N� )�util�Popenc @ sH e Zd ZdZdd� Zdd� Zejfdd�Zdd d �Z d d � Z d d� Z dS )r �forkc C s t j� d | _| j|� d S )N)r Z_flush_std_streams� returncode�_launch)�self� process_obj� r �2/usr/lib64/python3.6/multiprocessing/popen_fork.py�__init__ s zPopen.__init__c C s |S )Nr )r �fdr r r �duplicate_for_child s zPopen.duplicate_for_childc C s� | j d kr�x>ytj| j|�\}}W n tk rB } zd S d }~X q X P q W || jkr�tj|�rntj|� | _ ntj|�s|t�tj |�| _ | j S )N) r �os�waitpid�pid�OSError� WIFSIGNALED�WTERMSIG� WIFEXITED�AssertionError� WEXITSTATUS)r �flagr �sts�er r r �poll s z Popen.pollNc C sN | j d krH|d k r0ddlm} || jg|�s0d S | j|dkrBtjnd�S | j S )Nr )�waitg )r Zmultiprocessing.connectionr �sentinelr r �WNOHANG)r �timeoutr r r r r + s z Popen.waitc C s\ | j d krXytj| jtj� W n8 tk r2 Y n& tk rV | jdd�d krR� Y nX d S )Ng�������?)r ) r r �killr �signal�SIGTERM�ProcessLookupErrorr r )r r r r � terminate5 s zPopen.terminatec C s� d}t j� \}}t j� | _| jdkrdz0t j|� dtjkrJdd l}|j� |j � }W d t j |� X n"t j|� t j | t j|f� || _ d S )Nr r �random)r �piper r �close�sys�modulesr% Zseed� _bootstrap�_exitr ZFinalizer )r r �codeZparent_rZchild_wr% r r r r ? s z Popen._launch)N) �__name__� __module__� __qualname__�methodr r r r r r r$ r r r r r r s )r r( r! � r �__all__�objectr r r r r �<module> s __pycache__/spawn.cpython-36.opt-2.pyc 0000644 00000013442 15204243010 0013465 0 ustar 00 3 \�" � @ s. d dl Z d dlZd dlZd dlZddlmZmZ ddlmZ ddlm Z ddlm Z ddd d d d d gZ ej dkrzdZ dZn&ej dko�eedd�Z ejj� jd�Zer�e jjejd�anejadd � Zdd � Zdd� Zdd� Zdd � Zd&dd�Z dd� Z d d � Z d d � Z g Z d d � Z!d!d"� Z"d#d$� Z#d%d � Z$dS )'� N� )�get_start_method�set_start_method)�process)� reduction)�util�_main�freeze_support�set_executable�get_executable�get_preparation_data�get_command_line�import_main_path�win32F�frozenzpythonservice.exez python.exec C s | a d S )N)� _python_exe)Zexe� r �-/usr/lib64/python3.6/multiprocessing/spawn.pyr ) s c C s t S )N)r r r r r r - s c C s$ t | �dkr | d dkr dS dS d S )N� r z--multiprocessing-forkTF)�len)�argvr r r � is_forking4 s r c C sh t tj�rdi } xBtjdd � D ]0}|jd�\}}|dkrBd | |< q t|�| |< q W tf | � tj� d S )Nr �=�None)r �sysr �split�int� spawn_main�exit)�kwds�arg�name�valuer r r r > s c K sh t tdd�r(tjdgdd� | j� D � S d}|djdd � | j� D ��; }tj� }tg| d |dg S d S ) Nr Fz--multiprocessing-forkc S s g | ] }d | �qS )z%s=%rr )�.0�itemr r r � <listcomp>T s z$get_command_line.<locals>.<listcomp>z<from multiprocessing.spawn import spawn_main; spawn_main(%s)z, c s s | ]}d | V qdS )z%s=%rNr )r# r$ r r r � <genexpr>W s z#get_command_line.<locals>.<genexpr>z-c)�getattrr � executable�items�joinr Z _args_from_interpreter_flagsr )r �progZoptsr r r r N s c C s\ t jdkr.dd l}tj|| �}|j|tj�}nddlm } ||j _ | }t |�}t j |� d S )Nr r r )�semaphore_tracker)r �platform�msvcrtr Z steal_handleZopen_osfhandle�os�O_RDONLY� r, Z_semaphore_trackerZ_fdr r )Z pipe_handleZ parent_pidZ tracker_fdr. Z new_handle�fdr, Zexitcoder r r r \ s r c C s^ t j| ddd��@}dtj� _z$tjj|�}t|� tjj|�}W d tj� `X W d Q R X |j � S )N�rbT)�closefd) r/ �fdopenr �current_process� _inheritingr �pickle�load�prepare� _bootstrap)r2 Z from_parentZpreparation_data�selfr r r r m s c C s t tj� dd�rtd��d S )Nr7 Fa An attempt has been made to start a new process before the current process has finished its bootstrapping phase. This probably means that you are not using fork to start your child processes and you have forgotten to use the proper idiom in the main module: if __name__ == '__main__': freeze_support() ... The "freeze_support()" line can be omitted if the program is not going to be frozen to produce an executable.)r' r r6 � RuntimeErrorr r r r �_check_not_importing_mainy s r> c C s t � ttjtj� jd�}tjd k r2tjj� |d< t j j � }y|j d�}W n t k r^ Y n X tj||< |j| |t jtjtj� t� d� t jd }t|jdd �}|d k r�||d< nht jdks�t o�t �r t|d d �}|d k �r tj j|� o�tjd k �r tj jtj|�}tj j |�|d < |S ) N)� log_to_stderr�authkey� log_levelr1 )r! �sys_path�sys_argv�orig_dir�dir� start_method�__main__r! �init_main_from_namer �__file__�init_main_from_path) r> �dictr Z_log_to_stderrr r6 r@ Z_loggerZgetEffectiveLevelr �path�copy�index� ValueError� ORIGINAL_DIR�updater r/ �getcwdr �modulesr' �__spec__r- �WINEXE� WINSERVICE�isabsr* �normpath)r! �drB �i� main_moduleZ main_mod_name� main_pathr r r r � s> c C s� d| kr| d t j� _d| kr,| d t j� _d| krD| d rDtj� d| kr^tj� j| d � d| krp| d t_ d| kr�| d t_ d| kr�t j | d � d| kr�| d t _ d | kr�t| d d d � d | kr�t| d � nd | kr�t| d � d S )Nr! r@ r? rA rB rC rE rD rF T)ZforcerH rJ )r r6 r! r@ r r? Z get_loggerZsetLevelr rL r r/ �chdirrP r �_fixup_main_from_name�_fixup_main_from_path)�datar r r r: � s, r: c C s~ t jd }| dks | jd�r d S t|jdd �| kr6d S tj|� tjd�}t j | ddd�}|j j |� | t jd< t jd< d S )NrG z .__main__r! � __mp_main__T)�run_nameZ alter_sys) r rS �endswithr' rT �old_main_modules�append�types� ModuleType�runpyZ run_module�__dict__rQ )Zmod_name� current_mainr[ � main_contentr r r r^ � s r^ c C s� t jd }tjjtjj| ��d }|dkr.d S t|dd �| krBd S tj|� t j d�}t j | dd�}|j j|� | t jd< t jd< d S )NrG r ZipythonrI ra )rb )r rS r/ rL �splitext�basenamer' rd re rf rg rh Zrun_pathri rQ )r\ rj Z main_namer[ rk r r r r_ � s r_ c C s t | � d S )N)r_ )r\ r r r r s )NN)%r/ r rh rf r1 r r r �contextr r �__all__r- rU rV r' r( �lowerrc rL r* � exec_prefixr r r r r r r r r> r rd r: r^ r_ r r r r r �<module> s> 2& __pycache__/pool.cpython-36.pyc 0000644 00000050527 15204243010 0012353 0 ustar 00 3 \�e � @ s8 d dgZ ddlZddlZddlZddlZddlZddlZddlZddlm Z ddlm Z m Z dZ dZ dZej� Zdd � Zd d � ZG d d � d e�ZG dd� d�Zdd� ZG dd� de�Zdf ddfdd�Zdd� ZG dd � d e�ZG dd� de�Z e Z G d d � d e �Z G d d � d e�Z G d d!� d!e �Z G d"d� de�Z!dS )#�Pool� ThreadPool� N� )�util)� get_context� TimeoutError� c C s t t| � �S )N)�list�map)�args� r �,/usr/lib64/python3.6/multiprocessing/pool.py�mapstar+ s r c C s t tj| d | d ��S )Nr r )r � itertools�starmap)r r r r � starmapstar. s r c @ s e Zd Zdd� Zdd� ZdS )�RemoteTracebackc C s || _ d S )N)�tb)�selfr r r r �__init__6 s zRemoteTraceback.__init__c C s | j S )N)r )r r r r �__str__8 s zRemoteTraceback.__str__N)�__name__� __module__� __qualname__r r r r r r r 5 s r c @ s e Zd Zdd� Zdd� ZdS )�ExceptionWithTracebackc C s0 t jt|�||�}dj|�}|| _d| | _d S )N� z """ %s""")� traceback�format_exception�type�join�excr )r r r r r r r < s z ExceptionWithTraceback.__init__c C s t | j| jffS )N)� rebuild_excr r )r r r r � __reduce__A s z!ExceptionWithTraceback.__reduce__N)r r r r r" r r r r r ; s r c C s t |�| _| S )N)r � __cause__)r r r r r r! D s r! c s0 e Zd ZdZ� fdd�Zdd� Zdd� Z� ZS )�MaybeEncodingErrorzVWraps possible unpickleable errors, so they can be safely sent through the socket.c s. t |�| _t |�| _tt| �j| j| j� d S )N)�reprr �value�superr$ r )r r r& )� __class__r r r P s zMaybeEncodingError.__init__c C s d| j | jf S )Nz(Error sending result: '%s'. Reason: '%s')r& r )r r r r r U s zMaybeEncodingError.__str__c C s d| j j| f S )Nz<%s: %s>)r( r )r r r r �__repr__Y s zMaybeEncodingError.__repr__)r r r �__doc__r r r) � __classcell__r r )r( r r$ L s r$ Fc ' C s� |d ks t |�tkr |dks t�|j}| j}t| d�rJ| jj� |jj� |d k rZ||� d}�xL|d ksx|ot||k �r�y |� } W n$ t t fk r� t j d� P Y nX | d kr�t j d� P | \} } } } }yd| | |�f}W nF t k �r" } z(|o�| tk �r t||j�}d|f}W Y d d }~X nX y|| | |f� W nR t k �r� } z4t||d �}t j d| � || | d|ff� W Y d d }~X nX d } } } } } }|d7 }qbW t j d | � d S ) Nr �_writerz)worker got EOFError or OSError -- exitingz worker got sentinel -- exitingTFr z0Possible encoding error while sending result: %sz worker exiting after %d tasks)r �int�AssertionError�put�get�hasattrr, �close�_reader�EOFError�OSErrorr �debug� Exception�_helper_reraises_exceptionr � __traceback__r$ )�inqueue�outqueue� initializer�initargsZmaxtasksZwrap_exceptionr/ r0 Z completed�task�job�i�funcr �kwds�result�e�wrappedr r r �worker] sF $ rF c C s | �dS )z@Pickle-able helper function for use by _guarded_task_generation.Nr )Zexr r r r8 � s r8 c @ s6 e Zd ZdZdZdd� Zddf ddfdd�Zdd � Zd d � Zd d � Z dd� Z f i fdd�Z d=dd�Z d>dd�Z d?dd�Zdd� Zd@dd �ZdAd d �Zf i ddfd d �ZdBd!d"�ZdCd#d$�Zed%d&� �Zed'd(� �Zed)d*� �Zed+d,� �Zd-d.� Zd/d0� Zd1d2� Z d3d4� Z ed5d6� �Z e d7d8� �Z d9d:� Z!d;d<� Z"dS )Dr zS Class which supports an async version of applying functions to arguments. Tc O s | j j||�S )N)�_ctx�Process)r r rB r r r rH � s z Pool.ProcessNc C sn |pt � | _| j� tj� | _i | _t| _|| _ || _ || _ |d krPt j � pNd}|dk r`td��|d k rzt|� rztd��|| _g | _| j� tjtj| fd�| _d| j_t| j_| jj� tjtj| j| j | j | j| jfd�| _ d| j _t| j _| j j� tjtj | j | j | jfd�| _!d| j!_t| j!_| j!j� t"j#| | j$| j| j%| j | j| j| j | j!| jfdd�| _&d S )Nr z&Number of processes must be at least 1z initializer must be a callable)�targetr T� )r Z exitpriority)'r rG � _setup_queues�queue�Queue� _taskqueue�_cache�RUN�_state�_maxtasksperchild� _initializer� _initargs�os� cpu_count� ValueError�callable� TypeError� _processes�_pool�_repopulate_pool� threadingZThreadr �_handle_workers�_worker_handler�daemon�start� _handle_tasks� _quick_put� _outqueue� _task_handler�_handle_results� _quick_get�_result_handlerr ZFinalize�_terminate_pool�_inqueue� _terminate)r � processesr< r= Zmaxtasksperchild�contextr r r r � sT z Pool.__init__c C sZ d}xPt tt| j���D ]:}| j| }|jdk rtjd| � |j� d}| j|= qW |S )z�Cleanup after any worker processes which have exited due to reaching their specified lifetime. Returns True if any workers were cleaned up. FNzcleaning up worker %dT)�reversed�range�lenr[ �exitcoder r6 r )r Zcleanedr@ rF r r r �_join_exited_workers� s zPool._join_exited_workersc C s| xvt | jt| j� �D ]^}| jt| j| j| j| j | j | j fd�}| jj |� |j jdd�|_ d|_|j� tjd� qW dS )z�Bring the number of pool processes up to the specified number, for use after reaping workers which have exited. )rI r rH Z PoolWorkerTz added workerN)ro rZ rp r[ rH rF rj rd rS rT rR �_wrap_exception�append�name�replacer` ra r r6 )r r@ �wr r r r\ � s zPool._repopulate_poolc C s | j � r| j� dS )zEClean up any exited workers and start replacements for them. N)rr r\ )r r r r �_maintain_pool� s zPool._maintain_poolc C s4 | j j� | _| j j� | _| jjj| _| jjj| _ d S )N) rG Z SimpleQueuerj rd r, �sendrc r3 �recvrg )r r r r rK � s zPool._setup_queuesc C s | j tkst�| j|||�j� S )z6 Equivalent of `func(*args, **kwds)`. )rQ rP r. � apply_asyncr0 )r rA r rB r r r �apply� s z Pool.applyc C s | j ||t|�j� S )zx Apply `func` to each element in `iterable`, collecting the results in a list that is returned. )� _map_asyncr r0 )r rA �iterable� chunksizer r r r s zPool.mapc C s | j ||t|�j� S )z� Like `map()` method but the elements of the `iterable` are expected to be iterables as well and will be unpacked as arguments. Hence `func` and (a, b) becomes func(a, b). )r} r r0 )r rA r~ r r r r r s z Pool.starmapc C s | j ||t|||�S )z= Asynchronous version of `starmap()` method. )r} r )r rA r~ r �callback�error_callbackr r r � starmap_async s zPool.starmap_asyncc c sn y0d}x&t |�D ]\}}||||fi fV qW W n8 tk rh } z ||d t|fi fV W Y dd}~X nX dS )z�Provides a generator of tasks for imap and imap_unordered with appropriate handling for iterables which throw exceptions during iteration.r N���)� enumerater7 r8 )r Z result_jobrA r~ r@ �xrD r r r �_guarded_task_generation s z Pool._guarded_task_generationr c C s� | j tkrtd��|dkrFt| j�}| jj| j|j||�|j f� |S |dksRt �t j |||�}t| j�}| jj| j|jt |�|j f� dd� |D �S dS )zP Equivalent of `map()` -- can be MUCH slower than `Pool.map()`. zPool not runningr c s s | ]}|D ] }|V q qd S )Nr )�.0�chunk�itemr r r � <genexpr>@ s z Pool.imap.<locals>.<genexpr>N)rQ rP rW � IMapIteratorrO rN r/ r� �_job� _set_lengthr. r � _get_tasksr )r rA r~ r rC � task_batchesr r r �imap' s" z Pool.imapc C s� | j tkrtd��|dkrFt| j�}| jj| j|j||�|j f� |S |dksRt �t j |||�}t| j�}| jj| j|jt |�|j f� dd� |D �S dS )zL Like `imap()` method but ordering of results is arbitrary. zPool not runningr c s s | ]}|D ] }|V q qd S )Nr )r� r� r� r r r r� [ s z&Pool.imap_unordered.<locals>.<genexpr>N)rQ rP rW �IMapUnorderedIteratorrO rN r/ r� r� r� r. r r� r )r rA r~ r rC r� r r r �imap_unorderedB s" zPool.imap_unorderedc C sB | j tkrtd��t| j||�}| jj|jd|||fgdf� |S )z; Asynchronous version of `apply()` method. zPool not runningr N)rQ rP rW � ApplyResultrO rN r/ r� )r rA r rB r� r� rC r r r r{ ] s zPool.apply_asyncc C s | j ||t|||�S )z9 Asynchronous version of `map()` method. )r} r )r rA r~ r r� r� r r r � map_asynch s zPool.map_asyncc C s� | j tkrtd��t|d�s$t|�}|dkrTtt|�t| j�d �\}}|rT|d7 }t|�dkrdd}tj |||�}t | j |t|�||d�} | j j | j| j||�df� | S )zY Helper function to implement map, starmap and their async counterparts. zPool not running�__len__N� r r )r� )rQ rP rW r1 r �divmodrp r[ r r� � MapResultrO rN r/ r� r� ) r rA r~ Zmapperr r� r� Zextrar� rC r r r r} p s& zPool._map_asyncc C sT t j� }x0|jtks$| jr8|jtkr8| j� tjd� q W | j j d � t j d� d S )Ng�������?zworker handler exiting) r] �current_threadrQ rP rO � TERMINATErx �time�sleeprN r/ r r6 )�pool�threadr r r r^ � s zPool._handle_workersc C sj t j� }x�t| jd �D ]�\}}d }z�x�|D ]�}|jr@tjd� P y ||� W q* tk r� } zD|d d� \} } y|| j| d| f� W n t k r� Y nX W Y d d } ~ X q*X q*W |r�tjd� |r�|d nd } || d � wP W d d } }} X qW tjd� y:tjd� |j d � tjd� x|D ]} |d � �q&W W n t k �rZ tjd � Y nX tjd � d S ) Nz'task handler found thread._state != RUNr Fzdoing set_length()r ztask handler got sentinelz/task handler sending sentinel to result handlerz(task handler sending sentinel to workersz/task handler got OSError when sending sentinelsztask handler exitingr� ) r] r� �iterr0 rQ r r6 r7 �_set�KeyErrorr/ r5 ) � taskqueuer/ r; r� �cacher� ZtaskseqZ set_lengthr> rD r? �idx�pr r r rb � sB zPool._handle_tasksc C s� t j� }x�y |� }W n" ttfk r6 tjd� d S X |jrX|jtksLt�tjd� P |d krltjd� P |\}}}y|| j ||� W n t k r� Y nX d } }}q W x�|o�|jtk�rJy |� }W n" ttfk r� tjd� d S X |d k�rtjd� q�|\}}}y|| j ||� W n t k �r: Y nX d } }}q�W t | d��r�tjd� y,x&t d�D ]}| j j� �s�P |� �qnW W n ttfk �r� Y nX tjdt|�|j� d S ) Nz.result handler got EOFError/OSError -- exitingz,result handler found thread._state=TERMINATEzresult handler got sentinelz&result handler ignoring extra sentinelr3 z"ensuring that outqueue is not full� z7result handler exiting: len(cache)=%s, thread._state=%s)r] r� r5 r4 r r6 rQ r� r. r� r� r1 ro r3 �pollrp )r; r0 r� r� r> r? r@ �objr r r rf � s\ zPool._handle_resultsc c s4 t |�}x&ttj||��}|s"d S | |fV q W d S )N)r� �tupler �islice)rA �it�sizer� r r r r� s zPool._get_tasksc C s t d��d S )Nz:pool objects cannot be passed between processes or pickled)�NotImplementedError)r r r r r" s zPool.__reduce__c C s& t jd� | jtkr"t| _t| j_d S )Nz closing pool)r r6 rQ rP �CLOSEr_ )r r r r r2 s z Pool.closec C s$ t jd� t| _t| j_| j� d S )Nzterminating pool)r r6 r� rQ r_ rk )r r r r � terminate s zPool.terminatec C sV t jd� | jttfks t�| jj� | jj� | j j� x| j D ] }|j� qBW d S )Nz joining pool) r r6 rQ r� r� r. r_ r re rh r[ )r r� r r r r s z Pool.joinc C sD t jd� | jj� x*|j� r>| jj� r>| jj� tj d� qW d S )Nz7removing tasks from inqueue until task handler finishedr ) r r6 Z_rlock�acquire�is_aliver3 r� rz r� r� )r: � task_handlerr� r r r �_help_stuff_finish( s zPool._help_stuff_finishc C sF t jd� t|_t|_t jd� | j||t|�� |j� sJt|�dksJt�t|_|jd � t jd� t j � |k rx|j � |r�t |d d�r�t jd� x |D ]} | j d kr�| j� q�W t jd� t j � |k r�|j � t jd� t j � |k r�|j � |�rBt |d d��rBt jd � x0|D ](} | j� �rt jd | j � | j � �qW d S ) Nzfinalizing poolz&helping task handler/workers to finishr zjoining worker handlerr� zterminating workerszjoining task handlerzjoining result handlerzjoining pool workerszcleaning up worker %d)r r6 r� rQ r� rp r� r. r/ r] r� r r1 rq r� �pid) �clsr� r: r; r� Zworker_handlerr� Zresult_handlerr� r� r r r ri 1 s8 zPool._terminate_poolc C s | S )Nr )r r r r � __enter___ s zPool.__enter__c C s | j � d S )N)r� )r �exc_typeZexc_valZexc_tbr r r �__exit__b s z Pool.__exit__)N)N)NNN)r )r )NNN)NNN)#r r r r* rs rH r rr r\ rx rK r| r r r� r� r� r� r{ r� r} � staticmethodr^ rb rf r� r" r2 r� r r� � classmethodri r� r� r r r r r � sF 8 . < .c @ s@ e Zd Zdd� Zdd� Zdd� Zddd �Zdd d �Zd d � ZdS )r� c C s4 t j� | _tt�| _|| _|| _|| _| || j< d S )N) r] ZEvent�_event�next� job_counterr� rO � _callback�_error_callback)r r� r� r� r r r r k s zApplyResult.__init__c C s | j j� S )N)r� Zis_set)r r r r �readys s zApplyResult.readyc C s | j � s t�| jS )N)r� r. �_success)r r r r � successfulv s zApplyResult.successfulNc C s | j j|� d S )N)r� �wait)r �timeoutr r r r� z s zApplyResult.waitc C s, | j |� | j� st�| jr"| jS | j�d S )N)r� r� r r� �_value)r r� r r r r0 } s zApplyResult.getc C sV |\| _ | _| jr$| j r$| j| j� | jr>| j r>| j| j� | jj� | j| j= d S )N)r� r� r� r� r� �setrO r� )r r@ r� r r r r� � s zApplyResult._set)N)N) r r r r r� r� r� r0 r� r r r r r� i s r� c @ s e Zd Zdd� Zdd� ZdS )r� c C sf t j| |||d� d| _d g| | _|| _|dkrLd| _| jj� || j= n|| t || � | _d S )N)r� Tr ) r� r r� r� � _chunksize� _number_leftr� r� r� �bool)r r� r �lengthr� r� r r r r � s zMapResult.__init__c C s� | j d8 _ |\}}|rp| jrp|| j|| j |d | j �< | j dkr�| jrZ| j| j� | j| j= | jj� nH| r�| jr�d| _|| _| j dkr�| j r�| j | j� | j| j= | jj� d S )Nr r F) r� r� r� r� r� rO r� r� r� r� )r r@ Zsuccess_result�successrC r r r r� � s" zMapResult._setN)r r r r r� r r r r r� � s r� c @ s: e Zd Zdd� Zdd� Zd dd�ZeZdd � Zd d � ZdS ) r� c C sJ t jt j� �| _tt�| _|| _tj � | _ d| _ d | _ i | _ | || j< d S )Nr )r] Z ConditionZLock�_condr� r� r� rO � collections�deque�_items�_index�_length� _unsorted)r r� r r r r � s zIMapIterator.__init__c C s | S )Nr )r r r r �__iter__� s zIMapIterator.__iter__Nc C s� | j �~ y| jj� }W nh tk r~ | j| jkr6t�| j j|� y| jj� }W n( tk rx | j| jkrpt�t�Y nX Y nX W d Q R X |\}}|r�|S |�d S )N) r� r� �popleft� IndexErrorr� r� � StopIterationr� r )r r� r� r� r& r r r r� � s" zIMapIterator.nextc C s� | j �� | j|krr| jj|� | jd7 _x8| j| jkrd| jj| j�}| jj|� | jd7 _q.W | j j� n || j|< | j| jkr�| j| j = W d Q R X d S )Nr ) r� r� r� rt r� �pop�notifyr� rO r� )r r@ r� r r r r� � s zIMapIterator._setc C s< | j �, || _| j| jkr.| j j� | j| j= W d Q R X d S )N)r� r� r� r� rO r� )r r� r r r r� � s zIMapIterator._set_length)N) r r r r r� r� �__next__r� r� r r r r r� � s r� c @ s e Zd Zdd� ZdS )r� c C sP | j �@ | jj|� | jd7 _| j j� | j| jkrB| j| j= W d Q R X d S )Nr )r� r� rt r� r� r� rO r� )r r@ r� r r r r� s zIMapUnorderedIterator._setN)r r r r� r r r r r� � s r� c @ s@ e Zd ZdZedd� �Zddf fdd�Zdd� Zed d � �ZdS ) r Fc O s ddl m} || |�S )Nr )rH )ZdummyrH )r rB rH r r r rH s zThreadPool.ProcessNc C s t j| |||� d S )N)r r )r rl r< r= r r r r s zThreadPool.__init__c C s, t j� | _t j� | _| jj| _| jj| _d S )N)rL rM rj rd r/ rc r0 rg )r r r r rK s zThreadPool._setup_queuesc C s< | j �, | jj� | jjd g| � | j j� W d Q R X d S )N)Z not_emptyrL �clear�extendZ notify_all)r: r� r� r r r r� s z ThreadPool._help_stuff_finish) r r r rs r� rH r rK r� r r r r r s )"�__all__r] rL r r� rU r� r r r r r rP r� r� �countr� r r r7 r r r! r$ rF r8 �objectr r� Z AsyncResultr� r� r� r r r r r �<module> s@ * \&)@ __pycache__/popen_forkserver.cpython-36.opt-2.pyc 0000644 00000004430 15204243010 0015723 0 ustar 00 3 \� � @ s� d dl Z d dlZddlmZmZ ejs.ed��ddlmZ ddlm Z ddlm Z ddlm Z d gZ G d d � d e �ZG d d � d e j�ZdS ) � N� )� reduction�set_spawning_popenz,No support for sending fds between processes)� forkserver)� popen_fork)�spawn)�util�Popenc @ s e Zd Zdd� Zdd� ZdS )�_DupFdc C s || _ d S )N)�ind)�selfr � r �8/usr/lib64/python3.6/multiprocessing/popen_forkserver.py�__init__ s z_DupFd.__init__c C s t j� | j S )N)r Zget_inherited_fdsr )r r r r �detach s z _DupFd.detachN)�__name__� __module__� __qualname__r r r r r r r s r c sB e Zd ZdZeZ� fdd�Zdd� Zdd� Ze j fdd �Z � Z S ) r r c s g | _ t� j|� d S )N)�_fds�superr )r � process_obj)� __class__r r r ! s zPopen.__init__c C s | j j|� t| j �d S )Nr )r �append�len)r �fdr r r �duplicate_for_child% s zPopen.duplicate_for_childc C s� t j|j�}tj� }t| � z tj||� tj||� W d td � X tj | j �\| _ }t j | tj| j f� t|ddd��}|j|j� � W d Q R X tj| j �| _d S )N�wbT)�closefd)r Zget_preparation_data�_name�io�BytesIOr r �dumpr Zconnect_to_new_processr �sentinelr ZFinalize�os�close�open�write� getbuffer� read_unsigned�pid)r r Z prep_dataZbuf�w�fr r r �_launch) s z Popen._launchc C sr | j d krlddlm} |tjkr$dnd }|| jg|�s:d S ytj| j�| _ W n tt fk rj d| _ Y nX | j S )Nr )�wait� ) � returncodeZmultiprocessing.connectionr- r# �WNOHANGr" r r( �OSError�EOFError)r �flagr- Ztimeoutr r r �poll9 s z Popen.poll) r r r �methodr ZDupFdr r r, r# r0 r4 � __classcell__r r )r r r s )r r# �contextr r ZHAVE_SEND_HANDLE� ImportError� r r r r �__all__�objectr r r r r r �<module> s __pycache__/sharedctypes.cpython-36.pyc 0000644 00000015334 15204243010 0014075 0 ustar 00 3 \e � @ s: d dl Z d dlZddlmZ ddlmZ ddlmZmZ ejZ dddd d d gZ e j e j e j e je je je je je je je je jd � Zd d� Zdd� Zdd� Zddd�dd�Zddd�dd �Z dd � Z d&dd �Z dd� Z dd� Z dd � Z!d Z"i Z#ej$� Z%G d d � d e&�Z'G d d!� d!e'�Z(G d"d#� d#e'�Z)G d$d%� d%e)�Z*dS )'� N� )�heap)� get_context)� reduction�assert_spawning�RawValue�RawArray�Value�Array�copy� synchronized) �c�u�b�B�h�H�i�I�l�L�f�dc C s t j| �}tj|�}t| |d �S )N)�ctypes�sizeofr Z BufferWrapper� rebuild_ctype)�type_�size�wrapper� r �4/usr/lib64/python3.6/multiprocessing/sharedctypes.py� _new_value& s r! c G s< t j| | �}t|�}tjtj|�dtj|�� |j|� |S )z> Returns a ctypes object allocated from shared memory r )�typecode_to_type�getr! r �memset� addressofr �__init__)�typecode_or_type�argsr �objr r r r + s c C sj t j| | �}t|t�rD|| }t|�}tjtj|�dtj|�� |S |t |� }t|�}|j |� |S dS )z= Returns a ctypes array allocated from shared memory r N) r" r# � isinstance�intr! r r$ r% r �lenr& )r' �size_or_initializerr r) �resultr r r r 5 s T)�lock�ctxc G sX t | f|�� }|dkr|S |dkr4|p*t� }|j� }t|d�sJtd| ��t|||d�S )z6 Return a synchronization wrapper for a Value FTN�acquirez '%r' has no method 'acquire')r0 )TN)r r �RLock�hasattr�AttributeErrorr )r' r/ r0 r( r) r r r r E s c C sT t | |�}|dkr|S |dkr0|p&t� }|j� }t|d�sFtd| ��t|||d�S )z9 Return a synchronization wrapper for a RawArray FTNr1 z '%r' has no method 'acquire')r0 )TN)r r r2 r3 r4 r )r' r- r/ r0 r) r r r r S s c C s t t| ��}| tj|�d< |S )Nr )r! �typer Zpointer)r) Znew_objr r r r a s c C s� t | t� std��|p t� }t | tj�r6t| ||�S t | tj�rf| jtj krZt | ||�S t | ||�S t | �}y t | }W nV tk r� dd� |jD �}tdd� |D ��}d|j }t |tf|� }t |< Y nX || ||�S d S )Nzobject already synchronizedc S s g | ] }|d �qS )r r )�.0Zfieldr r r � <listcomp>u s z synchronized.<locals>.<listcomp>c s s | ]}|t |�fV qd S )N)� make_property)r6 �namer r r � <genexpr>v s z synchronized.<locals>.<genexpr>� Synchronized)r* �SynchronizedBase�AssertionErrorr r Z _SimpleCDatar; r �_type_�c_char�SynchronizedString�SynchronizedArrayr5 � class_cache�KeyErrorZ_fields_�dict�__name__)r) r/ r0 �cls�scls�namesr Z classnamer r r r f s" c C s@ t | � t| tj�r(t| j| j| jffS tt| �| jd ffS d S )N) r r* r r r r> �_wrapperZ_length_r5 )r) r r r � reduce_ctype s rJ c C s8 |d k r| | } t j| t� |j� }| j|�}||_|S )N)�_ForkingPickler�registerrJ Zcreate_memoryviewZ from_bufferrI )r r ZlengthZbufr) r r r r � s r c C sJ yt | S tk rD i }tt| fd |� || t | < || S X d S )N� )� prop_cacherC �exec�template)r9 r r r r r8 � s r8 z� def get%s(self): self.acquire() try: return self._obj.%s finally: self.release() def set%s(self, value): self.acquire() try: self._obj.%s = value finally: self.release() %s = property(get%s, set%s) c @ sF e Zd Zddd�Zdd� Zdd� Zdd � Zd d � Zd d � Zdd� Z dS )r< Nc C sB || _ |r|| _n|p tdd�}|j� | _| jj| _| jj| _d S )NT)Zforce)�_obj�_lockr r2 r1 �release)�selfr) r/ r0 r r r r& � s zSynchronizedBase.__init__c C s | j j� S )N)rR � __enter__)rT r r r rU � s zSynchronizedBase.__enter__c G s | j j|� S )N)rR �__exit__)rT r( r r r rV � s zSynchronizedBase.__exit__c C s t | � t| j| jffS )N)r r rQ rR )rT r r r � __reduce__� s zSynchronizedBase.__reduce__c C s | j S )N)rQ )rT r r r �get_obj� s zSynchronizedBase.get_objc C s | j S )N)rR )rT r r r �get_lock� s zSynchronizedBase.get_lockc C s dt | �j| jf S )Nz<%s wrapper for %s>)r5 rE rQ )rT r r r �__repr__� s zSynchronizedBase.__repr__)NN) rE � __module__� __qualname__r& rU rV rW rX rY rZ r r r r r< � s r< c @ s e Zd Zed�ZdS )r; �valueN)rE r[ r\ r8 r] r r r r r; � s r; c @ s4 e Zd Zdd� Zdd� Zdd� Zdd� Zd d � Zd S ) rA c C s t | j�S )N)r, rQ )rT r r r �__len__� s zSynchronizedArray.__len__c C s | � | j | S Q R X d S )N)rQ )rT r r r r � __getitem__� s z SynchronizedArray.__getitem__c C s | � || j |< W d Q R X d S )N)rQ )rT r r] r r r � __setitem__� s z SynchronizedArray.__setitem__c C s | � | j ||� S Q R X d S )N)rQ )rT �start�stopr r r � __getslice__� s z SynchronizedArray.__getslice__c C s" | � || j ||�< W d Q R X d S )N)rQ )rT ra rb �valuesr r r � __setslice__� s z SynchronizedArray.__setslice__N)rE r[ r\ r^ r_ r` rc re r r r r rA � s rA c @ s e Zd Zed�Zed�ZdS )r@ r] �rawN)rE r[ r\ r8 r] rf r r r r r@ � s r@ )NN)+r �weakref� r r �contextr r ZForkingPicklerrK �__all__r? Zc_wcharZc_byteZc_ubyteZc_shortZc_ushortZc_intZc_uintZc_longZc_ulongZc_floatZc_doubler" r! r r r r r r rJ r r8 rP rN �WeakKeyDictionaryrB �objectr< r; rA r@ r r r r �<module> s: __pycache__/popen_spawn_win32.cpython-36.pyc 0000644 00000005310 15204243010 0014743 0 ustar 00 3 \� � @ s� d dl Z d dlZd dlZd dlZd dlZddlmZmZmZ ddl m Z ddl m Z dgZ dZ ejdkoreed d �Zejj� jd �ZG d d� de�ZdS ) � N� )� reduction�get_spawning_popen�set_spawning_popen)�spawn)�util�Popeni Zwin32�frozenFzpythonservice.exec @ s> e Zd ZdZdZdd� Zdd� Zddd �Zd d � Zd d � Z dS )r z@ Start a subprocess to run the code of a process object r c C s t j|j�}tjd d�\}}tj|d�}t jtj � |d�}dj dd� |D ��}t |ddd���}y4tj t j � |d d d dd d d � \}} } } tj| � W n tj|� � Y nX | | _d | _|| _t|�| _tj| tj| jf� t| � z tj||� tj||� W d td � X W d Q R X d S ) Nr )Z parent_pidZ pipe_handle� c s s | ]}d | V qdS )z"%s"N� )�.0�xr r �9/usr/lib64/python3.6/multiprocessing/popen_spawn_win32.py� <genexpr>) s z!Popen.__init__.<locals>.<genexpr>�wbT)�closefdF)r Zget_preparation_data�_name�_winapiZ CreatePipe�msvcrtZopen_osfhandleZget_command_line�os�getpid�join�openZ CreateProcessZget_executableZ CloseHandle�pid� returncode�_handle�int�sentinelr ZFinalizer r �dump) �selfZ process_objZ prep_dataZrhandleZwhandleZwfd�cmdZto_childZhpZhtr �tidr r r �__init__ s2 zPopen.__init__c C s | t � kst�tj|| j�S )N)r �AssertionErrorr Z duplicater )r Zhandler r r �duplicate_for_childE s zPopen.duplicate_for_childNc C st | j d krn|d krtj}ntdt|d d ��}tjt| j�|�}|tjkrntj| j�}|t krht j }|| _ | j S )Nr i� g �?) r r ZINFINITE�maxr ZWaitForSingleObjectr Z WAIT_OBJECT_0ZGetExitCodeProcess� TERMINATE�signal�SIGTERM)r �timeoutZmsecs�res�coder r r �waitI s z Popen.waitc C s | j dd�S )Nr )r) )r, )r r r r �pollY s z Popen.pollc C sL | j d krHytjt| j�t� W n& tk rF | jdd�d krB� Y nX d S )Ng �?)r) )r r ZTerminateProcessr r r& �OSErrorr, )r r r r � terminate\ s zPopen.terminate)N) �__name__� __module__� __qualname__�__doc__�methodr" r$ r, r- r/ r r r r r s % )r r r' �sysr �contextr r r � r r �__all__r& �platform�getattrZWINEXE� executable�lower�endswithZ WINSERVICE�objectr r r r r �<module> s __pycache__/context.cpython-36.opt-1.pyc 0000644 00000031452 15204243010 0014021 0 ustar 00 3 \�* � @ s� d dl Z d dlZd dlZddlmZ ddlmZ g ZG dd� de�ZG dd� de�Z G d d � d e�Z G d d � d e�Z G d d� de �Z G dd� dej�ZG dd� de �Zedd� ee�D ��e_ejdk�rPG dd� dej�ZG dd� dej�ZG dd� dej�ZG d d � d e �ZG d d � d e �ZG d d!� d!e �Ze� e� e� d"�Zeed# �Zn8G d$d� dej�ZG d%d � d e �Zd&e� iZeed& �Zd'd(� Z ej � Z d)d*� Z d+d,� Z d-d.� Z!dS )/� N� )�process)� reductionc @ s e Zd ZdS )� ProcessErrorN)�__name__� __module__� __qualname__� r r �//usr/lib64/python3.6/multiprocessing/context.pyr s r c @ s e Zd ZdS )�BufferTooShortN)r r r r r r r r s r c @ s e Zd ZdS )� TimeoutErrorN)r r r r r r r r s r c @ s e Zd ZdS )�AuthenticationErrorN)r r r r r r r r s r c @ sV e Zd ZeZeZeZeZeej �Z eej �Z dd� Z dd� Z dBdd�Z dd � Zd d � ZdCd d�ZdDdd�ZdEdd�Zdd� ZdFdd�ZdGdd�ZdHdd �Zd d � Zd d f d fd d �Zd!d"� Zd#d$� Zdd%�d&d'�Zdd%�d(d)�Z d*d+� Z d,d-� Z dId.d/�Z d0d1� Z d2d3� Z!d4d5� Z"dJd6d7�Z#dKd9d:�Z$dLd;d<�Z%e&d=d>� �Z'e'j(d?d>� �Z'd@dA� Z)d S )M� BaseContextc C s" t j� }|dkrtd��n|S dS )z(Returns the number of CPUs in the systemNz cannot determine number of cpus)�os� cpu_count�NotImplementedError)�selfZnumr r r r ( s zBaseContext.cpu_countc C s& ddl m} || j� d�}|j� |S )z�Returns a manager associated with a running server process The managers methods such as `Lock()`, `Condition()` and `Queue()` can be used to create shared objects. r )� SyncManager)�ctx)Zmanagersr � get_context�start)r r �mr r r �Manager0 s zBaseContext.ManagerTc C s ddl m} ||�S )z1Returns two connection object connected by a piper )�Pipe)� connectionr )r Zduplexr r r r r ; s zBaseContext.Pipec C s ddl m} || j� d�S )z#Returns a non-recursive lock objectr )�Lock)r )� synchronizer r )r r r r r r @ s zBaseContext.Lockc C s ddl m} || j� d�S )z Returns a recursive lock objectr )�RLock)r )r r r )r r r r r r E s zBaseContext.RLockNc C s ddl m} ||| j� d�S )zReturns a condition objectr )� Condition)r )r r r )r �lockr r r r r J s zBaseContext.Conditionr c C s ddl m} ||| j� d�S )zReturns a semaphore objectr )� Semaphore)r )r r r )r �valuer r r r r O s zBaseContext.Semaphorec C s ddl m} ||| j� d�S )z"Returns a bounded semaphore objectr )�BoundedSemaphore)r )r r" r )r r! r" r r r r" T s z BaseContext.BoundedSemaphorec C s ddl m} || j� d�S )zReturns an event objectr )�Event)r )r r# r )r r# r r r r# Y s zBaseContext.Eventc C s ddl m} ||||| j� d�S )zReturns a barrier objectr )�Barrier)r )r r$ r )r Zparties�actionZtimeoutr$ r r r r$ ^ s zBaseContext.Barrierr c C s ddl m} ||| j� d�S )zReturns a queue objectr )�Queue)r )�queuesr& r )r �maxsizer& r r r r& c s zBaseContext.Queuec C s ddl m} ||| j� d�S )zReturns a queue objectr )� JoinableQueue)r )r' r) r )r r( r) r r r r) h s zBaseContext.JoinableQueuec C s ddl m} || j� d�S )zReturns a queue objectr )� SimpleQueue)r )r' r* r )r r* r r r r* m s zBaseContext.SimpleQueuec C s" ddl m} |||||| j� d�S )z Returns a process pool objectr )�Pool)�context)Zpoolr+ r )r Z processesZ initializerZinitargsZmaxtasksperchildr+ r r r r+ r s zBaseContext.Poolc G s ddl m} ||f|�� S )zReturns a shared objectr )�RawValue)� sharedctypesr- )r �typecode_or_type�argsr- r r r r- y s zBaseContext.RawValuec C s ddl m} |||�S )zReturns a shared arrayr )�RawArray)r. r1 )r r/ �size_or_initializerr1 r r r r1 ~ s zBaseContext.RawArray)r c G s& ddl m} ||f|�|| j� d��S )z$Returns a synchronized shared objectr )�Value)r r )r. r3 r )r r/ r r0 r3 r r r r3 � s zBaseContext.Valuec C s ddl m} ||||| j� d�S )z#Returns a synchronized shared arrayr )�Array)r r )r. r4 r )r r/ r2 r r4 r r r r4 � s zBaseContext.Arrayc C s, t jdkr(tt dd�r(ddlm} |� dS )z�Check whether this is a fake forked process in a frozen executable. If so then run code specified by commandline and exit. �win32�frozenFr )�freeze_supportN)�sys�platform�getattr�spawnr7 )r r7 r r r r7 � s zBaseContext.freeze_supportc C s ddl m} |� S )zZReturn package logger -- if it does not already exist then it is created. r )� get_logger)�utilr< )r r< r r r r< � s zBaseContext.get_loggerc C s ddl m} ||�S )z8Turn on logging and add a handler which prints to stderrr )� log_to_stderr)r= r> )r �levelr> r r r r> � s zBaseContext.log_to_stderrc C s ddl m} dS )zVInstall support for sending connections and sockets between processes r )r N)� r )r r r r r �allow_connection_pickling� s z%BaseContext.allow_connection_picklingc C s ddl m} ||� dS )z�Sets the path to a python.exe or pythonw.exe binary used to run child processes instead of sys.executable when using the 'spawn' start method. Useful for people embedding Python. r )�set_executableN)r; rB )r � executablerB r r r rB � s zBaseContext.set_executablec C s ddl m} ||� dS )zkSet list of module names to try to load in forkserver process. This is really just a hint. r )�set_forkserver_preloadN)� forkserverrD )r Z module_namesrD r r r rD � s z"BaseContext.set_forkserver_preloadc C sF |d kr | S y t | }W n tk r8 td| ��Y nX |j� |S )Nzcannot find context for %r)�_concrete_contexts�KeyError� ValueError�_check_available)r �methodr r r r r � s zBaseContext.get_contextFc C s | j S )N)�_name)r � allow_noner r r �get_start_method� s z BaseContext.get_start_methodc C s t d��d S )Nz+cannot set start method of concrete context)rH )r rJ �forcer r r �set_start_method� s z BaseContext.set_start_methodc C s t � jd�S )z_Controls how objects will be reduced to a form that can be shared with other processes.r )�globals�get)r r r r �reducer� s zBaseContext.reducerc C s |t � d< d S )Nr )rP )r r r r r rR � s c C s d S )Nr )r r r r rI � s z BaseContext._check_available)T)N)r )r )NN)r )r )N)N)F)F)*r r r r r r r � staticmethodr Zcurrent_processZactive_childrenr r r r r r r r" r# r$ r& r) r* r+ r- r1 r3 r4 r7 r< r> rA rB rD r rM rO �propertyrR �setterrI r r r r r sJ r c @ s e Zd ZdZedd� �ZdS )�ProcessNc C s t j� jj| �S )N)�_default_contextr rV �_Popen)� process_objr r r rX � s zProcess._Popen)r r r � _start_methodrS rX r r r r rV � s rV c sF e Zd ZeZdd� Zd � fdd� Zddd�Zdd d �Zd d � Z� Z S )�DefaultContextc C s || _ d | _d S )N)rW �_actual_context)r r, r r r �__init__� s zDefaultContext.__init__Nc s0 |d kr | j d kr| j| _ | j S t� j|�S d S )N)r\ rW �superr )r rJ )� __class__r r r � s zDefaultContext.get_contextFc C s> | j d k r| rtd��|d kr.|r.d | _ d S | j|�| _ d S )Nz context has already been set)r\ � RuntimeErrorr )r rJ rN r r r rO � s z DefaultContext.set_start_methodc C s" | j d kr|rd S | j| _ | j jS )N)r\ rW rK )r rL r r r rM � s z DefaultContext.get_start_methodc C s, t jdkrdgS tjr dddgS ddgS d S )Nr5 r; �forkrE )r8 r9 r �HAVE_SEND_HANDLE)r r r r �get_all_start_methods� s z$DefaultContext.get_all_start_methods)N)F)F) r r r rV r] r rO rM rc � __classcell__r r )r_ r r[ � s r[ c c s | ]}|d dkr|V qdS )r �_Nr )�.0�xr r r � <genexpr> s rh r5 c @ s e Zd ZdZedd� �ZdS )� ForkProcessra c C s ddl m} || �S )Nr )�Popen)Z popen_forkrj )rY rj r r r rX s zForkProcess._PopenN)r r r rZ rS rX r r r r ri s ri c @ s e Zd ZdZedd� �ZdS )� SpawnProcessr; c C s ddl m} || �S )Nr )rj )Zpopen_spawn_posixrj )rY rj r r r rX s zSpawnProcess._PopenN)r r r rZ rS rX r r r r rk s rk c @ s e Zd ZdZedd� �ZdS )�ForkServerProcessrE c C s ddl m} || �S )Nr )rj )Zpopen_forkserverrj )rY rj r r r rX s zForkServerProcess._PopenN)r r r rZ rS rX r r r r rl s rl c @ s e Zd ZdZeZdS )� ForkContextra N)r r r rK ri rV r r r r rm % s rm c @ s e Zd ZdZeZdS )� SpawnContextr; N)r r r rK rk rV r r r r rn ) s rn c @ s e Zd ZdZeZdd� ZdS )�ForkServerContextrE c C s t jstd��d S )Nz%forkserver start method not available)r rb rH )r r r r rI 0 s z"ForkServerContext._check_availableN)r r r rK rl rV rI r r r r ro - s ro )ra r; rE ra c @ s e Zd ZdZedd� �ZdS )rk r; c C s ddl m} || �S )Nr )rj )Zpopen_spawn_win32rj )rY rj r r r rX ? s zSpawnProcess._PopenN)r r r rZ rS rX r r r r rk = s c @ s e Zd ZdZeZdS )rn r; N)r r r rK rk rV r r r r rn D s r; c C s t | t_d S )N)rF rW r\ )rJ r r r �_force_start_methodQ s rp c C s t tdd �S )N�spawning_popen)r: �_tlsr r r r �get_spawning_popenZ s rs c C s | t _d S )N)rr rq )�popenr r r �set_spawning_popen] s ru c C s t � d kr tdt| �j ��d S )NzF%s objects should only be shared between processes through inheritance)rs r` �typer )�objr r r �assert_spawning` s rx )"r r8 Z threadingr@ r r �__all__� Exceptionr r r r �objectr Z BaseProcessrV r[ �list�dirr9 ri rk rl rm rn ro rF rW rp Zlocalrr rs ru rx r r r r �<module> sD >' __pycache__/heap.cpython-36.opt-1.pyc 0000644 00000013405 15204243010 0013250 0 ustar 00 3 \ � @ s� d dl Z d dlZd dlZd dlZd dlZd dlZddlmZmZ ddl m Z dgZ ej dkrvd dl Z G dd� de�Zn,G d d� de�Zd d � Zd d � Zejee� G dd� de�ZG dd� de�ZdS )� N� )� reduction�assert_spawning)�util� BufferWrapperZwin32c @ s, e Zd Zej� Zdd� Zdd� Zdd� ZdS )�Arenac C sz || _ xTtd�D ]@}dtj� t| j�f }tjd||d�}tj� dkrHP |j � qW t d��|| _ || _ | j | j f| _ d S )N�d z pym-%d-%sr )�tagnamer z Cannot find name for new mmap���)�size�range�os�getpid�next�_rand�mmap�_winapiZ GetLastError�close�FileExistsError�name�buffer�_state)�selfr �ir Zbuf� r �,/usr/lib64/python3.6/multiprocessing/heap.py�__init__" s zArena.__init__c C s t | � | jS )N)r r )r r r r � __getstate__1 s zArena.__getstate__c C s, | \| _ | _| _tjd| j | jd�| _d S )Nr )r r )r r r r r )r �stater r r � __setstate__5 s zArena.__setstate__N) �__name__� __module__� __qualname__�tempfileZ_RandomNameSequencer r r r r r r r r s r c @ s e Zd Zddd�ZdS )r r c C s� || _ || _|d kr�tjdtj� tj� d�\| _}tj|� tj | tj | jf� t | jddd��N}d }||kr�d| }x t || �D ]}|j |� q�W ~|j d|| � W d Q R X tj| j| j �| _d S ) Nr zpym-%d-)�prefix�dir�wbF)�closefdi � r i )r �fdr# Zmkstempr r r Z get_temp_dir�unlink�Finalizer �openr �writer r )r r r) r �fZbsZzeros�_r r r r @ s" zArena.__init__Nr )r )r r! r" r r r r r r > s c C s( | j dkrtd��t| jtj| j �ffS )Nr zDArena is unpicklable because forking was enabled when it was createdr )r) � ValueError� rebuild_arenar r ZDupFd)�ar r r � reduce_arenaS s r3 c C s t | |j� �S )N)r �detach)r Zdupfdr r r r1 Y s r1 c @ sZ e Zd ZdZejfdd�Zedd� �Zdd� Z dd � Z d d � Z d d � Z dd� Z dd� ZdS )�Heap� c C sJ t j� | _tj� | _|| _g | _i | _i | _ i | _ t � | _ g | _ g | _d S )N)r r �_lastpid� threadingZLock�_lock�_size�_lengths� _len_to_seq�_start_to_block�_stop_to_block�set�_allocated_blocks�_arenas�_pending_free_blocks)r r r r r r f s z Heap.__init__c C s |d }| | | @ S )Nr r )�nZ alignment�maskr r r �_roundups s z Heap._roundupc C s� t j| j|�}|t| j�krj| jt| j|�tj�}| jd9 _t j d|� t |�}| j j |� |d|fS | j| }| j| }|j� }|s�| j|= | j|= |\}}}| j||f= | j||f= |S )N� z"allocating a new mmap of length %dr )�bisectZ bisect_leftr; �lenrE �maxr: r �PAGESIZEr �infor rA �appendr< �popr= r> ) r r r �length�arena�seq�block�start�stopr r r �_mallocy s" z Heap._mallocc C s� |\}}}y| j ||f }W n tk r0 Y nX | j|�\}}y| j||f }W n tk rf Y nX | j|�\}}|||f}|| }y| j| j|� W n. tk r� |g| j|< tj| j|� Y nX || j||f< || j ||f< d S )N) r> �KeyError�_absorbr= r< rL rG Zinsortr; ) r rQ rO rR rS Z prev_blockr/ Z next_blockrN r r r �_free� s( z Heap._freec C s^ |\}}}| j ||f= | j||f= || }| j| }|j|� |sV| j|= | jj|� ||fS )N)r= r> r< �remover; )r rQ rO rR rS rN rP r r r rV � s z Heap._absorbc C sF x@y| j j� }W n tk r& P Y nX | jj|� | j|� qW d S )N)rB rM � IndexErrorr@ rX rW )r rQ r r r �_free_pending_blocks� s zHeap._free_pending_blocksc C sN | j jd�s| jj|� n0z"| j� | jj|� | j|� W d | j j� X d S )NF) r9 �acquirerB rL rZ r@ rX rW �release)r rQ r r r �free� s z Heap.freec C s� t j� | jkr| j� | j�h | j� | jt|d�| j�}| j |�\}}}|| }||k rj| j |||f� |||f}| j j |� |S Q R X d S )Nr ) r r r7 r r9 rZ rE rI � _alignmentrT rW r@ �add)r r rO rR rS Znew_stoprQ r r r �malloc� s z Heap.mallocN)r r! r" r^ r rJ r � staticmethodrE rT rW rV rZ r] r` r r r r r5 b s r5 c @ s" e Zd Ze� Zdd� Zdd� ZdS )r c C s0 t jj|�}||f| _tj| t jj|fd� d S )N)�args)r �_heapr` r r r+ r] )r r rQ r r r r � s zBufferWrapper.__init__c C s& | j \\}}}}t|j�||| � S )N)r � memoryviewr )r rO rR rS r r r r �create_memoryview� s z BufferWrapper.create_memoryviewN)r r! r" r5 rc r re r r r r r � s )rG r r �sysr# r8 �contextr r � r �__all__�platformr �objectr r3 r1 �registerr5 r r r r r �<module> s$ __pycache__/connection.cpython-36.pyc 0000644 00000060406 15204243010 0013536 0 ustar 00 3 �I�iVy � @ s d dddgZ ddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl m Z ddl m Z m Z dd lmZ ejZy$ddlZdd lmZmZmZmZ W n$ ek r� ejd kr dZY nX d Zd ZdZej � Z dZ dgZ e ed��rdZ e dg7 Z ejd k�r dZ e dg7 Z efdd�Z!dd� Z"dd� Z#dd� Z$dd� Z%G d d � d �Z&e�rnG d d � d e&�Z'G d d!� d!e&�Z(G d"d� de)�Z*dPd#d �Z+ejd k�r�dQd%d�Z,n dRd&d�Z,G d'd(� d(e)�Z-d)d*� Z.ejd k�r�G d+d,� d,e)�Z/d-d.� Z0d/Z1d0Z2d1Z3d2Z4d3d4� Z5d5d6� Z6G d7d8� d8e)�Z7d9d:� Z8d;d<� Z9G d=d>� d>e*�Z:d?d@� Z;ejd k�rzdAdB� Z<ej=ej>hZ?dSdCd�Z@n,ddlAZAe eAdD��r�eAjBZCneAjDZCdTdEd�Z@ejd k�r�dFdG� ZEdHdI� ZFejGe(eE� dJdK� ZHdLdM� ZIejGe'eH� n dNdG� ZEdOdI� ZFejGe(eE� dS )U�Client�Listener�Pipe�wait� N� )�util)�AuthenticationError�BufferTooShort)� reduction)� WAIT_OBJECT_0�WAIT_ABANDONED_0� WAIT_TIMEOUT�INFINITE�win32i g 4@Zsha256�AF_INET�AF_UNIX�AF_PIPEc C s t j� | S )N)�time� monotonic)�timeout� r �2/usr/lib64/python3.6/multiprocessing/connection.py� _init_timeout? s r c C s t j� | kS )N)r r )�tr r r �_check_timeoutB s r c C sX | dkr d S | dkr&t jdtj� d�S | dkrLt jdtj� tt�f d d�S td ��d S ) z? Return an arbitrary free address for the given family r � localhostr r z listener-)�prefix�dirr z\\.\pipe\pyc-%d-%d-� zunrecognized familyN)r r ) �tempfileZmktempr Z get_temp_dir�os�getpid�next� _mmap_counter� ValueError)�familyr r r �arbitrary_addressI s r& c C sJ t jdkr | dkr td| ��t jdkrF| dkrFtt| �sFtd| ��dS )zD Checks if the family is valid for the current environment. r r z Family %s is not recognized.r N)�sys�platformr$ �hasattr�socket)r% r r r �_validate_familyW s r+ c C sJ t | �tkrdS t | �tkr*| jd�r*dS t | �tkr:dS td| ��dS )z] Return the types of the address This can be 'AF_INET', 'AF_UNIX', or 'AF_PIPE' r z\\r r z address type of %r unrecognizedN)�type�tuple�str� startswithr$ )�addressr r r � address_typec s r1 c @ s� e Zd ZdZd+dd�Zdd� Zdd� Zd d � Zd d � Zd d� Z e dd� �Z e dd� �Z e dd� �Z dd� Zdd� Zd,dd�Zd d � Zd-d d �Zd.d d!�Zd"d#� Zd/d%d&�Zd'd(� Zd)d*� ZdS )0�_ConnectionBaseNTc C sB |j � }|dk rtd��| r,| r,td��|| _|| _|| _d S )Nr zinvalid handlez6at least one of `readable` and `writable` must be True)� __index__r$ �_handle� _readable� _writable)�self�handle�readable�writabler r r �__init__y s z_ConnectionBase.__init__c C s | j d k r| j� d S )N)r4 �_close)r7 r r r �__del__� s z_ConnectionBase.__del__c C s | j d krtd��d S )Nzhandle is closed)r4 �OSError)r7 r r r � _check_closed� s z _ConnectionBase._check_closedc C s | j std��d S )Nzconnection is write-only)r5 r> )r7 r r r �_check_readable� s z _ConnectionBase._check_readablec C s | j std��d S )Nzconnection is read-only)r6 r> )r7 r r r �_check_writable� s z _ConnectionBase._check_writablec C s" | j rd| _n| j� td��d S )NFzbad message length)r6 r5 �closer> )r7 r r r �_bad_message_length� s z#_ConnectionBase._bad_message_lengthc C s | j dkS )z True if the connection is closedN)r4 )r7 r r r �closed� s z_ConnectionBase.closedc C s | j S )z"True if the connection is readable)r5 )r7 r r r r9 � s z_ConnectionBase.readablec C s | j S )z"True if the connection is writable)r6 )r7 r r r r: � s z_ConnectionBase.writablec C s | j � | jS )z+File descriptor or handle of the connection)r? r4 )r7 r r r �fileno� s z_ConnectionBase.filenoc C s$ | j dk r z | j� W dd| _ X dS )zClose the connectionN)r4 r< )r7 r r r rB � s z_ConnectionBase.closer c C s� | j � | j� t|�}|jdkr.tt|��}t|�}|dk rFtd��||k rVtd��|dkrh|| }n&|dk rztd��n|| |kr�td��| j|||| � � dS )z,Send the bytes data from a bytes-like objectr r zoffset is negativezbuffer length < offsetNzsize is negativez buffer length < offset + size)r? rA � memoryview�itemsize�bytes�lenr$ � _send_bytes)r7 �buf�offset�size�m�nr r r � send_bytes� s" z_ConnectionBase.send_bytesc C s$ | j � | j� | jtj|�� dS )zSend a (picklable) objectN)r? rA rJ �_ForkingPickler�dumps)r7 �objr r r �send� s z_ConnectionBase.sendc C sJ | j � | j� |dk r(|dk r(td��| j|�}|dkrB| j� |j� S )z7 Receive bytes data as a bytes object. Nr znegative maxlength)r? r@ r$ � _recv_bytesrC �getvalue)r7 Z maxlengthrK r r r � recv_bytes� s z_ConnectionBase.recv_bytesc C s� | j � | j� t|���}|j}|t|� }|dk r>td��n||krNtd��| j� }|j� }||| k rvt|j � ��|j d� |j ||| || | � � |S Q R X dS )zq Receive bytes data into a writeable bytes-like object. Return the number of bytes read. r znegative offsetzoffset too largeN) r? r@ rF rG rI r$ rU �tellr rV �seek�readinto)r7 rK rL rN rG Zbytesize�resultrM r r r �recv_bytes_into� s" z _ConnectionBase.recv_bytes_intoc C s& | j � | j� | j� }tj|j� �S )z Receive a (picklable) object)r? r@ rU rQ �loads� getbuffer)r7 rK r r r �recv� s z_ConnectionBase.recv� c C s | j � | j� | j|�S )z/Whether there is any input available to be read)r? r@ �_poll)r7 r r r r �poll s z_ConnectionBase.pollc C s | S )Nr )r7 r r r � __enter__ s z_ConnectionBase.__enter__c C s | j � d S )N)rB )r7 �exc_type� exc_value�exc_tbr r r �__exit__ s z_ConnectionBase.__exit__)TT)r N)N)r )r` )�__name__� __module__� __qualname__r4 r; r= r? r@ rA rC �propertyrD r9 r: rE rB rP rT rW r\ r_ rb rc rg r r r r r2 v s( r2 c @ sD e Zd ZdZdZejfdd�Zdd� Zddd �Z d d � Z d d � Z dS )�PipeConnectionz� Connection class based on a Windows named pipe. Overlapped I/O is used, so the handles must have been created with FILE_FLAG_OVERLAPPED. Fc C s || j � d S )N)r4 )r7 Z _CloseHandler r r r< s zPipeConnection._closec C s� t j| j|dd�\}}zHy,|t jkrBt j|jgdt�}|tksBt�W n |j � � Y nX W d |j d�\}}X |dks|t�|t |�ks�t�d S )NT)� overlappedFr ) �_winapiZ WriteFiler4 �ERROR_IO_PENDING�WaitForMultipleObjects�eventr r �AssertionError�cancel�GetOverlappedResultrI )r7 rK �ov�err�waitresZnwrittenr r r rJ s zPipeConnection._send_bytesNc C s | j rd| _ tj� S |d kr dnt|d�}y�tj| j|dd�\}}zHy,|tjkrntj|j gdt �}|t ksnt �W n |j � � Y nX W d |jd�\}}|dkr�tj� }|j|j� � |S |tjkr�| j||�S X W n: tk �r } z |jtjk� r�t�n� W Y d d }~X nX td��d S )NF� T)rm r z.shouldn't get here; expected KeyboardInterrupt)�_got_empty_message�io�BytesIO�minrn �ReadFiler4 ro rp rq r r rr rs rt �writer^ ZERROR_MORE_DATA�_get_more_datar> �winerror�ERROR_BROKEN_PIPE�EOFError� RuntimeError) r7 �maxsizeZbsizeru rv rw Znread�f�er r r rU * s6 zPipeConnection._recv_bytesc C s. | j stj| j�d dkr dS tt| g|��S )Nr T)ry rn � PeekNamedPiper4 �boolr )r7 r r r r ra J s zPipeConnection._pollc C s� |j � }tj� }|j|� tj| j�d }|dks6t�|d k rVt|�| |krV| j � tj | j|dd�\}}|j d�\}}|dks�t�||ks�t�|j|j � � |S )Nr r T)rm ) r^ rz r{ r~ rn r� r4 rr rI rC r} rt )r7 ru r� rK r� �leftrv Zrbytesr r r r P s z PipeConnection._get_more_data)N) rh ri rj �__doc__ry rn � CloseHandler< rJ rU ra r r r r r rl s rl c @ s| e Zd ZdZer,ejfdd�ZejZ ej Z ne j fdd�Ze jZ e jZ e fdd�Ze fdd�Zd d � Zdd d �Zdd� Zd S )� Connectionzo Connection class based on an arbitrary file descriptor (Unix only), or a socket handle (Windows). c C s || j � d S )N)r4 )r7 r< r r r r< g s zConnection._closec C s || j � d S )N)r4 )r7 r< r r r r< l s c C s< t |�}x.|| j|�}||8 }|dkr(P ||d � }q W d S )Nr )rI r4 )r7 rK r~ � remainingrO r r r �_sendq s zConnection._sendc C sf t j� }| j}|}xN|dkr`|||�}t|�}|dkrL||krDt�ntd��|j|� ||8 }qW |S )Nr z got end of file during message)rz r{ r4 rI r� r> r~ )r7 rM �readrK r8 r� �chunkrO r r r �_recvz s zConnection._recvc C sD t |�}tjd|�}|dkr2| j|� | j|� n| j|| � d S )Nz!ii @ )rI �structZpackr� )r7 rK rO �headerr r r rJ � s zConnection._send_bytesNc C s: | j d�}tjd|j� �\}|d k r0||kr0d S | j |�S )N� z!i)r� r� ZunpackrV )r7 r� rK rM r r r rU � s zConnection._recv_bytesc C s t | g|�}t|�S )N)r r� )r7 r �rr r r ra � s zConnection._poll)N)rh ri rj r� rn �_multiprocessingZ closesocketr< rT Z_writer_ Z_readr rB r~ r� r� r� rJ rU ra r r r r r� ` s r� c @ sR e Zd ZdZddd�Zdd� Zdd � Zed d � �Zed d � �Z d d� Z dd� Z dS )r z� Returns a listener object. This is a wrapper for a bound socket which is 'listening' for connections, or for a Windows named pipe. Nr c C sr |p|rt |�pt}|p t|�}t|� |dkr>t||�| _nt|||�| _|d k rht|t� rht d��|| _ d S )Nr z authkey should be a byte string) r1 �default_familyr& r+ � PipeListener� _listener�SocketListener� isinstancerH � TypeError�_authkey)r7 r0 r% �backlog�authkeyr r r r; � s zListener.__init__c C s> | j dkrtd��| j j� }| jr:t|| j� t|| j� |S )zz Accept a connection on the bound socket or named pipe of `self`. Returns a `Connection` object. Nzlistener is closed)r� r> �acceptr� �deliver_challenge�answer_challenge)r7 �cr r r r� � s zListener.acceptc C s | j }|dk r d| _ |j� dS )zA Close the bound socket or named pipe of `self`. N)r� rB )r7 Zlistenerr r r rB � s zListener.closec C s | j jS )N)r� �_address)r7 r r r �<lambda>� s zListener.<lambda>c C s | j jS )N)r� �_last_accepted)r7 r r r r� � s c C s | S )Nr )r7 r r r rc � s zListener.__enter__c C s | j � d S )N)rB )r7 rd re rf r r r rg � s zListener.__exit__)NNr N) rh ri rj r� r; r� rB rk r0 Z last_acceptedrc rg r r r r r � s c C sj |p t | �}t|� |dkr&t| �}nt| �}|dk rJt|t� rJtd��|dk rft||� t||� |S )z= Returns a connection to the address of a `Listener` r Nz authkey should be a byte string) r1 r+ � PipeClient� SocketClientr� rH r� r� r� )r0 r% r� r� r r r r � s Tc C sj | r>t j� \}}|jd� |jd� t|j� �}t|j� �}n$tj� \}}t|dd�}t|dd�}||fS )zL Returns pair of connection objects at either end of a pipe TF)r: )r9 )r* Z socketpair� setblockingr� �detachr �pipe)�duplex�s1�s2�c1�c2Zfd1Zfd2r r r r � s c C s� t d�}| r*tj}tjtjB }tt }}ntj}tj}dt }}tj||tjB tj B tj tj B tj B d||tj tj�}tj||dtjtjtjtj�}tj|tj dd� tj|dd�}|jd�\} } | dks�t�t|| d�} t|| d�} | | fS ) zL Returns pair of connection objects at either end of a pipe r r r NT)rm )r: )r9 )r& rn �PIPE_ACCESS_DUPLEX� GENERIC_READ� GENERIC_WRITE�BUFSIZEZPIPE_ACCESS_INBOUND�CreateNamedPipe�FILE_FLAG_OVERLAPPED� FILE_FLAG_FIRST_PIPE_INSTANCE�PIPE_TYPE_MESSAGE�PIPE_READMODE_MESSAGE� PIPE_WAIT�NMPWAIT_WAIT_FOREVER�NULL� CreateFile� OPEN_EXISTING�SetNamedPipeHandleState�ConnectNamedPipert rr rl ) r� r0 Zopenmode�accessZobsizeZibsizeZh1Zh2rm �_rv r� r� r r r r s4 c @ s* e Zd ZdZd dd�Zdd� Zdd� Zd S ) r� zO Representation of a socket which is bound to an address and listening r c C s� t j tt |��| _yRtjdkr2| jjt jt jd� | jjd� | jj |� | jj |� | jj � | _ W n t k r� | jj� � Y nX || _d | _|dkr�tj| tj|fdd�| _nd | _d S )N�posixr Tr r )�args� exitpriority)r* �getattr�_socketr �nameZ setsockoptZ SOL_SOCKETZ SO_REUSEADDRr� ZbindZlistenZ getsocknamer� r> rB Z_familyr� r �Finalize�unlink�_unlink)r7 r0 r% r� r r r r; < s$ zSocketListener.__init__c C s& | j j� \}| _|jd� t|j� �S )NT)r� r� r� r� r� r� )r7 �sr r r r� T s zSocketListener.acceptc C s0 z| j j� W d | j}|d k r*d | _|� X d S )N)r� rB r� )r7 r� r r r rB Y s zSocketListener.closeN)r )rh ri rj r� r; r� rB r r r r r� 8 s r� c C sD t | �}tjtt|���"}|jd� |j| � t|j� �S Q R X dS )zO Return a connection object connected to the socket given by `address` TN)r1 r* r� r� Zconnectr� r� )r0 r% r� r r r r� c s r� c @ s8 e Zd ZdZd dd�Zd dd�Zdd � Zed d � �ZdS )r� z0 Representation of a named pipe Nc C sL || _ | jdd�g| _d | _tjd| j � tj| tj| j| j fdd�| _ d S )NT)�firstz listener created with address=%rr )r� r� ) r� � _new_handle� _handle_queuer� r � sub_debugr� r� �_finalize_pipe_listenerrB )r7 r0 r� r r r r; w s zPipeListener.__init__Fc C sH t jt jB }|r|t jO }t j| j|t jt jB t jB t j t t t j t j �S )N) rn r� r� r� r� r� r� r� r� ZPIPE_UNLIMITED_INSTANCESr� r� r� )r7 r� �flagsr r r r� � s zPipeListener._new_handlec C s� | j j| j� � | j jd�}ytj|dd�}W n0 tk r^ } z|jtjkrN� W Y d d }~X n\X z<ytj |j gdt �}W n |j � tj |� � Y nX W d |jd�\}}|dks�t�X t|�S )Nr T)rm F)r� �appendr� �poprn r� r> r� Z ERROR_NO_DATArp rq r rs r� rt rr rl )r7 r8 ru r� �resr� rv r r r r� � s" zPipeListener.acceptc C s( t jd|� x| D ]}tj|� qW d S )Nz closing listener with address=%r)r r� rn r� )Zqueuer0 r8 r r r r� � s z$PipeListener._finalize_pipe_listener)N)F) rh ri rj r� r; r� r� � staticmethodr� r r r r r� s s r� c C s� t � }x~y6tj| d� tj| tjtjB dtjtjtjtj�}W n> t k r| } z"|j tj tj fksjt |�rl� W Y dd}~X qX P qW � tj|tjdd� t|�S )zU Return a connection object connected to the pipe given by `address` i� r N)r rn Z WaitNamedPiper� r� r� r� r� r� r> r� ZERROR_SEM_TIMEOUTZERROR_PIPE_BUSYr r� r� rl )r0 r �hr� r r r r� � s" r� � s #CHALLENGE#s #WELCOME#s #FAILURE#c C st dd l }t|t�st�tjt�}| jt| � |j ||t �j � }| j d�}||kr^| jt � n| jt� td��d S )Nr � zdigest received was wrong)�hmacr� rH rr r �urandom�MESSAGE_LENGTHrP � CHALLENGE�new�HMAC_DIGEST_NAME�digestrW �WELCOME�FAILUREr )� connectionr� r� �messager� �responser r r r� � s r� c C s� dd l }t|t�st�| jd�}|d tt�� tks@td| ��|tt�d � }|j||t�j � }| j |� | jd�}|t kr�t d��d S )Nr r� z message = %rzdigest sent was rejected) r� r� rH rr rW rI r� r� r� r� rP r� r )r� r� r� r� r� r� r r r r� � s r� c @ s$ e Zd Zdd� Zdd� Zdd� ZdS )�ConnectionWrapperc C s: || _ || _|| _x"dD ]}t||�}t| ||� qW d S )NrE rB rb rW rP )rE rB rb rW rP )�_conn�_dumps�_loadsr� �setattr)r7 �connrR r] �attrrS r r r r; � s zConnectionWrapper.__init__c C s | j |�}| jj|� d S )N)r� r� rP )r7 rS r� r r r rT � s zConnectionWrapper.sendc C s | j j� }| j|�S )N)r� rW r� )r7 r� r r r r_ � s zConnectionWrapper.recvN)rh ri rj r; rT r_ r r r r r� � s r� c C s t j| fd d d d�jd�S )Nr zutf-8)� xmlrpclibrR �encode)rS r r r � _xml_dumps� s r� c C s t j| jd��\\}}|S )Nzutf-8)r� r] �decode)r� rS �methodr r r � _xml_loads� s r� c @ s e Zd Zdd� ZdS )� XmlListenerc C s dd l jatj| �}t|tt�S )Nr )� xmlrpc.client�clientr� r r� r� r� r� )r7 rS r r r r� s zXmlListener.acceptN)rh ri rj r� r r r r r� s r� c O s dd l jatt| |�tt�S )Nr )r� r� r� r� r r� r� )r� �kwdsr r r � XmlClient s r� c C s� t | �}g }x�|r�tj|d|�}|tkr,P n\t| koFtt|� k n rV|t8 }n2t| koptt|� k n r�|t8 }ntd��|j|| � ||d d � }d}qW |S )NFzShould not get herer r ) �listrn rp r r rI r r� r� )Zhandlesr �L�readyr� r r r �_exhaustive_wait s r� c . sn |dkrt }n |dk r d}nt|d d �}t| �} i �g }t� � t� }�zH�x2| D �](}yt|d�}W n tk r� |�|j� < Y qTX ytj|� dd�\}}W n8 t k r� } z d|j }}|t krʂ W Y dd}~X nX |tj kr�|j |� |�|j< qT|�rptj� dd� dk�rpy|jd �\} }W n* t k �rT } z |j }W Y dd}~X nX | �rpt|d ��rpd|_� j|� d}qTW t�j� |�}W dx|D ]}|j� �q�W x�|D ]�}y|jd�\} }W n6 t k �r� } z|j }|t k�r� W Y dd}~X nX |tjk�r��|j }� j|� |dk�r�t|d ��r�d|_�q�W X � j�fd d �|D �� � fd d�| D �S )z� Wait till an object in object_list is ready/readable. Returns list of those objects in object_list which are ready/readable. Nr i� g �?rE T� � Fry c 3 s | ]}� | V qd S )Nr )�.0r� )�waithandle_to_objr r � <genexpr>v s zwait.<locals>.<genexpr>c s g | ]}|� kr|�qS r r )r� �o)� ready_objectsr r � <listcomp>w s zwait.<locals>.<listcomp>)r� r� )r �intr� �setr� �AttributeErrorr3 rn r} r> r� � _ready_errorsro r� rq r' Zgetwindowsversionrt r) ry �addr� �keysrs ZERROR_OPERATION_ABORTED�update) � object_listr Zov_listZ ready_handlesr rE ru rv r� r� r )r r� r r + sh � PollSelectorc C s� t � �x}x| D ]}|j|tj� qW |dk r8tj� | }x@|j|�}|rVdd� |D �S |dk r:|tj� }|dk r:|S q:W W dQ R X dS )z� Wait till an object in object_list is ready/readable. Returns list of those objects in object_list which are ready/readable. Nc S s g | ]\}}|j �qS r )Zfileobj)r� �keyZeventsr r r r � s zwait.<locals>.<listcomp>r )� _WaitSelector�register� selectorsZ EVENT_READr r Zselect)r r ZselectorrS Zdeadliner� r r r r � s c C sN | j � }tj|tjtj��*}ddlm} |j|�}t|| j | j ffS Q R X d S )Nr )�resource_sharer) rE r* Zfromfdr Z SOCK_STREAMr r Z DupSocket�rebuild_connectionr9 r: )r� r8 r� r �dsr r r �reduce_connection� s r c C s | j � }t|j � ||�S )N)r� r� )r r9 r: Zsockr r r r � s r c C sB | j r tjnd| jrtjndB }tj| j� |�}t|| j | jffS )Nr ) r9 rn ZFILE_GENERIC_READr: ZFILE_GENERIC_WRITEr Z DupHandlerE �rebuild_pipe_connection)r� r� �dhr r r �reduce_pipe_connection� s r c C s | j � }t|||�S )N)r� rl )r r9 r: r8 r r r r � s r c C s t j| j� �}t|| j| jffS )N)r ZDupFdrE r r9 r: )r� �dfr r r r � s c C s | j � }t|||�S )N)r� r� )r r9 r: �fdr r r r � s )NN)T)T)N)N)J�__all__rz r r' r* r� r r � itertoolsr� r r r r �contextr ZForkingPicklerrQ rn r r r r � ImportErrorr( r� ZCONNECTION_TIMEOUTr� �countr# r� Zfamiliesr) r r r& r+ r1 r2 rl r� �objectr r r r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� ZERROR_NETNAME_DELETEDr r r r r ZSelectSelectorr r r r r r r r r �<module> s� PJ8 ,+ 8 P __pycache__/__init__.cpython-36.pyc 0000644 00000000773 15204243010 0013137 0 ustar 00 3 \� � @ s\ d dl Z ddlmZ e� jdd� ejjD �� ejjZdZdZde j krXe j d e j d < dS ) � N� )�contextc c s | ]}|t tj|�fV qd S )N)�getattrr �_default_context)�.0�name� r �0/usr/lib64/python3.6/multiprocessing/__init__.py� <genexpr> s r � � �__main__Z __mp_main__) �sys� r �globals�updater �__all__ZSUBDEBUGZ SUBWARNING�modulesr r r r �<module> s __pycache__/sharedctypes.cpython-36.opt-1.pyc 0000644 00000015231 15204243010 0015030 0 ustar 00 3 \e � @ s: d dl Z d dlZddlmZ ddlmZ ddlmZmZ ejZ dddd d d gZ e j e j e j e je je je je je je je je jd � Zd d� Zdd� Zdd� Zddd�dd�Zddd�dd �Z dd � Z d&dd �Z dd� Z dd� Z dd � Z!d Z"i Z#ej$� Z%G d d � d e&�Z'G d d!� d!e'�Z(G d"d#� d#e'�Z)G d$d%� d%e)�Z*dS )'� N� )�heap)� get_context)� reduction�assert_spawning�RawValue�RawArray�Value�Array�copy� synchronized) �c�u�b�B�h�H�i�I�l�L�f�dc C s t j| �}tj|�}t| |d �S )N)�ctypes�sizeofr Z BufferWrapper� rebuild_ctype)�type_�size�wrapper� r �4/usr/lib64/python3.6/multiprocessing/sharedctypes.py� _new_value& s r! c G s< t j| | �}t|�}tjtj|�dtj|�� |j|� |S )z> Returns a ctypes object allocated from shared memory r )�typecode_to_type�getr! r �memset� addressofr �__init__)�typecode_or_type�argsr �objr r r r + s c C sj t j| | �}t|t�rD|| }t|�}tjtj|�dtj|�� |S |t |� }t|�}|j |� |S dS )z= Returns a ctypes array allocated from shared memory r N) r" r# � isinstance�intr! r r$ r% r �lenr& )r' �size_or_initializerr r) �resultr r r r 5 s T)�lock�ctxc G sX t | f|�� }|dkr|S |dkr4|p*t� }|j� }t|d�sJtd| ��t|||d�S )z6 Return a synchronization wrapper for a Value FTN�acquirez '%r' has no method 'acquire')r0 )TN)r r �RLock�hasattr�AttributeErrorr )r' r/ r0 r( r) r r r r E s c C sT t | |�}|dkr|S |dkr0|p&t� }|j� }t|d�sFtd| ��t|||d�S )z9 Return a synchronization wrapper for a RawArray FTNr1 z '%r' has no method 'acquire')r0 )TN)r r r2 r3 r4 r )r' r- r/ r0 r) r r r r S s c C s t t| ��}| tj|�d< |S )Nr )r! �typer Zpointer)r) Znew_objr r r r a s c C s� |pt � }t| tj�r"t| ||�S t| tj�rR| jtjkrFt| ||�S t | ||�S t | �}y t | }W nV t k r� dd� |j D �}tdd� |D ��}d|j }t |tf|� }t |< Y nX || ||�S d S )Nc S s g | ] }|d �qS )r r )�.0Zfieldr r r � <listcomp>u s z synchronized.<locals>.<listcomp>c s s | ]}|t |�fV qd S )N)� make_property)r6 �namer r r � <genexpr>v s z synchronized.<locals>.<genexpr>� Synchronized)r r* r Z _SimpleCDatar; r �_type_�c_char�SynchronizedString�SynchronizedArrayr5 � class_cache�KeyErrorZ_fields_�dict�__name__�SynchronizedBase)r) r/ r0 �cls�scls�namesr Z classnamer r r r f s c C s@ t | � t| tj�r(t| j| j| jffS tt| �| jd ffS d S )N) r r* r r r r< �_wrapperZ_length_r5 )r) r r r � reduce_ctype s rI c C s8 |d k r| | } t j| t� |j� }| j|�}||_|S )N)�_ForkingPickler�registerrI Zcreate_memoryviewZ from_bufferrH )r r ZlengthZbufr) r r r r � s r c C sJ yt | S tk rD i }tt| fd |� || t | < || S X d S )N� )� prop_cacherA �exec�template)r9 r r r r r8 � s r8 z� def get%s(self): self.acquire() try: return self._obj.%s finally: self.release() def set%s(self, value): self.acquire() try: self._obj.%s = value finally: self.release() %s = property(get%s, set%s) c @ sF e Zd Zddd�Zdd� Zdd� Zdd � Zd d � Zd d � Zdd� Z dS )rD Nc C sB || _ |r|| _n|p tdd�}|j� | _| jj| _| jj| _d S )NT)Zforce)�_obj�_lockr r2 r1 �release)�selfr) r/ r0 r r r r& � s zSynchronizedBase.__init__c C s | j j� S )N)rQ � __enter__)rS r r r rT � s zSynchronizedBase.__enter__c G s | j j|� S )N)rQ �__exit__)rS r( r r r rU � s zSynchronizedBase.__exit__c C s t | � t| j| jffS )N)r r rP rQ )rS r r r � __reduce__� s zSynchronizedBase.__reduce__c C s | j S )N)rP )rS r r r �get_obj� s zSynchronizedBase.get_objc C s | j S )N)rQ )rS r r r �get_lock� s zSynchronizedBase.get_lockc C s dt | �j| jf S )Nz<%s wrapper for %s>)r5 rC rP )rS r r r �__repr__� s zSynchronizedBase.__repr__)NN) rC � __module__� __qualname__r& rT rU rV rW rX rY r r r r rD � s rD c @ s e Zd Zed�ZdS )r; �valueN)rC rZ r[ r8 r\ r r r r r; � s r; c @ s4 e Zd Zdd� Zdd� Zdd� Zdd� Zd d � Zd S ) r? c C s t | j�S )N)r, rP )rS r r r �__len__� s zSynchronizedArray.__len__c C s | � | j | S Q R X d S )N)rP )rS r r r r � __getitem__� s z SynchronizedArray.__getitem__c C s | � || j |< W d Q R X d S )N)rP )rS r r\ r r r � __setitem__� s z SynchronizedArray.__setitem__c C s | � | j ||� S Q R X d S )N)rP )rS �start�stopr r r � __getslice__� s z SynchronizedArray.__getslice__c C s" | � || j ||�< W d Q R X d S )N)rP )rS r` ra �valuesr r r � __setslice__� s z SynchronizedArray.__setslice__N)rC rZ r[ r] r^ r_ rb rd r r r r r? � s r? c @ s e Zd Zed�Zed�ZdS )r> r\ �rawN)rC rZ r[ r8 r\ re r r r r r> � s r> )NN)+r �weakref� r r �contextr r ZForkingPicklerrJ �__all__r= Zc_wcharZc_byteZc_ubyteZc_shortZc_ushortZc_intZc_uintZc_longZc_ulongZc_floatZc_doubler" r! r r r r r r rI r r8 rO rM �WeakKeyDictionaryr@ �objectrD r; r? r> r r r r �<module> s: __pycache__/__init__.cpython-36.opt-1.pyc 0000644 00000000773 15204243011 0014077 0 ustar 00 3 \� � @ s\ d dl Z ddlmZ e� jdd� ejjD �� ejjZdZdZde j krXe j d e j d < dS ) � N� )�contextc c s | ]}|t tj|�fV qd S )N)�getattrr �_default_context)�.0�name� r �0/usr/lib64/python3.6/multiprocessing/__init__.py� <genexpr> s r � � �__main__Z __mp_main__) �sys� r �globals�updater �__all__ZSUBDEBUGZ SUBWARNING�modulesr r r r �<module> s __pycache__/forkserver.cpython-36.opt-1.pyc 0000644 00000015126 15204243011 0014526 0 ustar 00 3 \�! � @ s� d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZddlm Z ddlm Z ddl m Z ddlm Z ddlmZ ddlmZ d d d d gZd Zejd�ZG dd� de�Zddd�Zdd� Zdd� Zdd� Ze� ZejZej Z ej Z ej Z dS )� N� )� connection)�process)� reduction)�semaphore_tracker)�spawn)�util�ensure_running�get_inherited_fds�connect_to_new_process�set_forkserver_preload� �Qc @ s4 e Zd Zdd� Zdd� Zdd� Zdd� Zd d � Zd S ) � ForkServerc C s. d | _ d | _d | _d | _tj� | _dg| _d S )N�__main__)�_forkserver_address�_forkserver_alive_fd�_forkserver_pid�_inherited_fds� threadingZLock�_lock�_preload_modules)�self� r �2/usr/lib64/python3.6/multiprocessing/forkserver.py�__init__! s zForkServer.__init__c C s& t dd� | jD ��s td��|| _dS )z>Set list of module names to try to load in forkserver process.c s s | ]}t |�tkV qd S )N)�type�str)�.0�modr r r � <genexpr>+ s z4ForkServer.set_forkserver_preload.<locals>.<genexpr>z&module_names must be a list of stringsN)�allr � TypeError)r Z modules_namesr r r r ) s z!ForkServer.set_forkserver_preloadc C s | j S )z�Return list of fds inherited from parent process. This returns None if the current process was not started by fork server. )r )r r r r r / s z ForkServer.get_inherited_fdsc C s� | j � t|�d tkr td��tjtj���}|j| j� tj � \}}tj � \}}||| j t j � g}||7 }z<yt j||� ||fS tj|� tj|� � Y nX W dtj|� tj|� X W dQ R X dS )a; Request forkserver to create a child process. Returns a pair of fds (status_r, data_w). The calling process can read the child process's pid and (eventually) its returncode from status_r. The calling process should write to data_w the pickled preparation and process data. � z too many fdsN)r �len�MAXFDS_TO_SEND� ValueError�socket�AF_UNIXZconnectr �os�piper r Zgetfdr Zsendfds�close)r �fdsZclientZparent_r�child_w�child_rZparent_wZallfdsr r r r 7 s& z!ForkServer.connect_to_new_processc s� | j ��n tj� | jdk rVtj| jtj�\}}|s8dS tj| j� d| _ d| _d| _d }| j r�ddh� t j d�}t � fdd�|j� D ��}ni }tjtj���}tjd �}|j|� tj|d � |j� tj� \}}ztyV|j� |g} ||j� || j |f; }t j� } | gtj� } | d |g7 } tj| | | �}W n tj|� � Y nX W dtj|� X || _ || _|| _W dQ R X W dQ R X dS ) z�Make sure that a fork server is running. This can be called from any process. Note that usually a child process will just reuse the forkserver started by its parent, so ensure_running() will do nothing. Nz-from multiprocessing.forkserver import main; zmain(%d, %d, %r, **%r)� main_path�sys_path�ignorec 3 s" | ]\}}|� kr||fV qd S )Nr )r �x�y)� desired_keysr r r o s z,ForkServer.ensure_running.<locals>.<genexpr>r( i� z-czCfrom multiprocessing.forkserver import main; main(%d, %d, %r, **%r)) r r r r r) �waitpid�WNOHANGr+ r r r r Zget_preparation_data�dict�itemsr' r( r Zarbitrary_addressZbind�chmodZlistenr* �filenoZget_executabler Z _args_from_interpreter_flagsZspawnv_passfds) r �pidZstatus�cmd�data�listenerZaddress�alive_rZalive_wZ fds_to_passZexe�argsr )r4 r r T sL zForkServer.ensure_runningN)�__name__� __module__� __qualname__r r r r r r r r r r s r c 4 C s� |rhd|kr8|dk r8dt j� _ztj|� W dt j� `X x.|D ]&}y t|� W q> tk rb Y q>X q>W tj� t j t j t j t j i}dd� |j � D �}tjtj| d���,}tj� ��} |j� t_| j|tj� | j|tj� x�y�x dd� | j� D �} | r�P q�W || k�r t�|j� d �l} d } tj� d k�r�zFyt | |||� W n. t k �rn t j t j � � t j!j"� Y nX W dtj#| � X W dQ R X W q� t$k �r� } z| j%t%j&k�r�� W Y dd} ~ X q�X q�W W dQ R X W dQ R X dS ) zRun forkserver.r NTc S s i | ]\}}t j ||�|�qS r )�signal)r �sig�valr r r � <dictcomp>� s zmain.<locals>.<dictcomp>)r: c S s g | ]\}}|j �qS r )Zfileobj)r �keyZeventsr r r � <listcomp>� s zmain.<locals>.<listcomp>r r )'r Zcurrent_processZ _inheritingr Zimport_main_path� __import__� ImportErrorr Z _close_stdinrD �SIGCHLD�SIG_IGN�SIGINTr8 r' r( � selectorsZDefaultSelectorZ getsockname� _forkserverr �registerZ EVENT_READZselect� SystemExitZacceptr) �fork� _serve_one� Exception�sys� excepthook�exc_info�stderr�flush�_exit�OSError�errnoZ ECONNABORTED)Z listener_fdr? Zpreloadr/ r0 �modname�handlersZ old_handlersr> ZselectorZrfds�s�code�er r r �main� sR rc c C s� |j � tj |� x |j� D ]\}}tj||� q W tj| td �}| j � |^}}t_} t_ | t j _ t |tj� � dtjkr�dd l} | j� tj|�} t || � d S )Nr �randomr )r+ r) r8 rD r Zrecvfdsr% rP r r r Z_semaphore_trackerZ_fd�write_unsigned�getpidrV �modulesrd Zseedr �_main) r` r> r? r_ rE rF r, r. r- Zstfdrd ra r r r rT � s rT c C sR d}t j}x8t|�|k rBtj| |t|� �}|s8td��||7 }q W t j|�d S )N� zunexpected EOFr )�UNSIGNED_STRUCT�sizer$ r) �read�EOFErrorZunpack)�fdr= Zlengthr` r r r � read_unsigned� s ro c C s@ t j|�}x0|r:tj| |�}|dkr,td��||d � }q W d S )Nr zshould not get here)rj Zpackr) �write� RuntimeError)rn �n�msg�nbytesr r r re � s re )NN) r] r) rO rD r' ZstructrV r � r r �contextr r r r �__all__r% ZStructrj �objectr rc rT ro re rP r r r r r r r r �<module> s6 s =! __pycache__/process.cpython-36.opt-1.pyc 0000644 00000016507 15204243011 0014020 0 ustar 00 3 \�# � @ s d ddgZ ddlZddlZddlZddlZddlmZ yejjej � �Z W n e k rb dZ Y nX dd� Z dd� Z dd � ZG d d � d e�ZG d d � d e�ZG d d� de�Ze� aejd�ae� a[i Zx<eejj � �D ]*\Z Z e dd� dko�de kr�e ee < q�W e� Z dS )� BaseProcess�current_process�active_children� N)�WeakSetc C s t S )z@ Return process object representing the current process )�_current_process� r r �//usr/lib64/python3.6/multiprocessing/process.pyr # s c C s t � tt�S )zN Return list of process objects corresponding to live child processes )�_cleanup�list� _childrenr r r r r ) s c C s. x(t t�D ] } | jj� d k r tj| � q W d S )N)r r �_popen�poll�discard)�pr r r r 4 s r c @ s� e Zd ZdZdd� Zdddf i fdd�dd�Zdd � Zd d � Zd d � Zd%dd�Z dd� Z e dd� �Z e j dd� �Z e dd� �Zej dd� �Ze dd� �Zej dd� �Ze dd � �Ze d d � �ZeZe d d � �Zd!d"� Zd#d$� ZdS )&r z� Process objects represent activity that is run in a separate process The class is analogous to `threading.Thread` c C s t �d S )N)�NotImplementedError)�selfr r r �_PopenD s zBaseProcess._PopenN)�daemonc C s� t t�}tj|f | _tjj� | _tj� | _d | _ || _ t |�| _ t |�| _|prt| �jd djdd� | jD �� | _|d k r�|| _tj| � d S )N�-�:c s s | ]}t |�V qd S )N)�str)�.0�ir r r � <genexpr>S s z'BaseProcess.__init__.<locals>.<genexpr>)�next�_process_counterr � _identity�_config�copy�os�getpid� _parent_pidr �_target�tuple�_args�dict�_kwargs�type�__name__�join�_namer � _dangling�add)r �group�target�name�args�kwargsr �countr r r �__init__G s zBaseProcess.__init__c C s | j r| j | j| j� dS )zQ Method to be run in sub-process; can be overridden in sub-class N)r" r$ r& )r r r r �runX s zBaseProcess.runc C s6 t � | j| �| _| jj| _| `| `| `tj | � dS )z% Start child process N) r r r �sentinel� _sentinelr" r$ r& r r, )r r r r �start_ s zBaseProcess.startc C s | j j� dS )zT Terminate process; sends SIGTERM signal or uses TerminateProcess() N)r � terminate)r r r r r8 p s zBaseProcess.terminatec C s" | j j|�}|dk r tj| � dS )z5 Wait until child process terminates N)r �waitr r )r Ztimeout�resr r r r) v s zBaseProcess.joinc C sB | t kr dS | jdkrdS | jj� }|dkr0dS tj| � dS dS )z1 Return whether process is alive TNF)r r r r r )r � returncoder r r �is_alive� s zBaseProcess.is_alivec C s | j S )N)r* )r r r r r/ � s zBaseProcess.namec C s || _ d S )N)r* )r r/ r r r r/ � s c C s | j jdd�S )z4 Return whether process is a daemon r F)r �get)r r r r r � s zBaseProcess.daemonc C s || j d< dS )z1 Set whether process is a daemon r N)r )r Zdaemonicr r r r � s c C s | j d S )N�authkey)r )r r r r r> � s zBaseProcess.authkeyc C s t |�| jd< dS )z2 Set authorization key of process r> N)�AuthenticationStringr )r r> r r r r> � s c C s | j dkr| j S | j j� S )zM Return exit code of process or `None` if it has yet to stop N)r r )r r r r �exitcode� s zBaseProcess.exitcodec C s" | t krtj� S | jo | jjS dS )zU Return identifier (PID) of process or `None` if it has yet to start N)r r r r �pid)r r r r �ident� s zBaseProcess.identc C s( y| j S tk r" td��Y nX dS )z{ Return a file descriptor (Unix) or handle (Windows) suitable for waiting for process termination. zprocess not startedN)r6 �AttributeError� ValueError)r r r r r5 � s zBaseProcess.sentinelc C s� | t krd}n>| jtj� kr"d}n*| jd kr2d}n| jj� d k rH| j}nd}t|�tkrv|dkrfd}ndt j ||� }dt| �j | j || j r�dp�d f S ) NZstarted�unknown�initialr Zstoppedz stopped[%s]z<%s(%s, %s%s)>z daemon� )r r! r r r r r@ r' �int�_exitcode_to_namer= r( r* r )r Zstatusr r r �__repr__� s zBaseProcess.__repr__c C sJ ddl m}m} �zy|| jd k r,|j| j� tjd�at� a |j � t }| a z|j j � |j� W d ~X |jd� z| j� d}W d |j� X W n� tk r� } zL|js�d}n:t|jd t�r�|jd }n tjjt|jd �d � d}W Y d d }~X n2 d}dd l}tjjd| j � |j � Y nX W d |jd| � |j � X |S )N� )�util�contextz child process calling self.run()r � z Process %s: z process exiting with exitcode %d) rG rL rM Z _start_methodZ_force_start_method� itertoolsr2 r �setr Z _close_stdinr Z_finalizer_registry�clearZ_run_after_forkers�infor4 Z_exit_function� SystemExitr0 � isinstancerH �sys�stderr�writer � tracebackr/ � print_excZ_flush_std_streams)r rL rM Z old_processr@ �erX r r r � _bootstrap� sB zBaseProcess._bootstrap)N)r( � __module__� __qualname__�__doc__r r3 r4 r7 r8 r) r< �propertyr/ �setterr r> r@ rB rA r5 rJ r[ r r r r r > s* c @ s e Zd Zdd� ZdS )r? c C s, ddl m} |� d kr td��tt| �ffS )NrK )�get_spawning_popenzJPickling an AuthenticationString object is disallowed for security reasons)rM ra � TypeErrorr? �bytes)r ra r r r � __reduce__ s z AuthenticationString.__reduce__N)r( r\ r] rd r r r r r? s r? c @ s e Zd Zdd� ZdS )� _MainProcessc C s2 f | _ d| _d | _d | _ttjd��dd�| _d S )NZ MainProcess� z/mp)r> Z semprefix)r r* r! r r? r �urandomr )r r r r r3 - s z_MainProcess.__init__N)r( r\ r] r3 r r r r re + s re rK � ZSIG�_) �__all__r rU �signalrO � _weakrefsetr �path�abspath�getcwdZ ORIGINAL_DIR�OSErrorr r r �objectr rc r? re r r2 r rP r rI r �__dict__�itemsr/ Zsignumr+ r r r r �<module> s2 ` __pycache__/resource_sharer.cpython-36.pyc 0000644 00000012044 15204243011 0014566 0 ustar 00 3 \� � @ s� d dl Z d dlZd dlZd dlZd dlZddlmZ ddlmZ ddlm Z dgZ ej dkrxe dg7 Z G d d� de �Z ne d g7 Z G d d � d e �ZG d d � d e �Ze� ZejZdS )� N� )�process)� reduction)�util�stopZwin32� DupSocketc @ s e Zd ZdZdd� Zdd� ZdS )r z Picklable wrapper for a socket.c s( |j � � � fdd�}tj|� j�| _d S )Nc s � j |�}| j|� d S )N)�shareZ send_bytes)�conn�pidr )�new_sock� �7/usr/lib64/python3.6/multiprocessing/resource_sharer.py�send s z DupSocket.__init__.<locals>.send)�dup�_resource_sharer�register�close�_id)�selfZsockr r )r r �__init__ s zDupSocket.__init__c C s* t j| j��}|j� }tj|�S Q R X dS )z1Get the socket. This should only be called once.N)r �get_connectionr Z recv_bytes�socketZ fromshare)r r r r r r �detach$ s zDupSocket.detachN)�__name__� __module__� __qualname__�__doc__r r r r r r r s �DupFdc @ s e Zd ZdZdd� Zdd� ZdS )r z-Wrapper for fd which can be used at any time.c s4 t j|�� � fdd�}� fdd�}tj||�| _d S )Nc s t j| � |� d S )N)r Z send_handle)r r )�new_fdr r r 1 s z DupFd.__init__.<locals>.sendc s t j� � d S )N)�osr r )r r r r 3 s z DupFd.__init__.<locals>.close)r r r r r )r �fdr r r )r r r / s zDupFd.__init__c C s" t j| j�� }tj|�S Q R X dS )z-Get the fd. This should only be called once.N)r r r r Z recv_handle)r r r r r r 7 s z DupFd.detachN)r r r r r r r r r r r - s c @ sN e Zd ZdZdd� Zdd� Zedd� �Zdd d �Zd d � Z d d� Z dd� Z dS )�_ResourceSharerz-Manager for resouces using background thread.c C s@ d| _ i | _g | _tj� | _d | _d | _d | _t j | t j � d S )Nr ) �_key�_cache� _old_locks� threading�Lock�_lock� _listener�_address�_threadr Zregister_after_forkr! � _afterfork)r r r r r ? s z_ResourceSharer.__init__c C sN | j �> | jdkr| j� | jd7 _||f| j| j< | j| jfS Q R X dS )z+Register resource, returning an identifier.Nr )r' r) �_startr" r# )r r r r r r r I s z_ResourceSharer.registerc C s< ddl m} | \}}||tj� jd�}|j|tj� f� |S )z<Return connection from which to receive identified resource.r )�Client)�authkey)� connectionr- r �current_processr. r r �getpid)Zidentr- �address�key�cr r r r R s z _ResourceSharer.get_connectionNc C s� ddl m} | j�� | jdk r�|| jtj� jd�}|jd� |j� | j j |� | j j � rdt j d� | jj� d| _ d| _d| _x | jj� D ]\}\}}|� q�W | jj� W dQ R X dS )z:Stop the background thread and clear registered resources.r )r- N)r. z._ResourceSharer thread did not stop when asked)r/ r- r' r) r r0 r. r r r* �joinZis_aliver Z sub_warningr( r# �items�clear)r Ztimeoutr- r4 r3 r r r r r r [ s" z_ResourceSharer.stopc C sn x | j j� D ]\}\}}|� q W | j j� | jj| j� tj� | _| jd k rX| jj � d | _d | _ d | _ d S )N) r# r6 r7 r$ �appendr' r% r& r( r r) r* )r r3 r r r r r r+ p s z_ResourceSharer._afterforkc C sf ddl m} | jd kst�tjd� |tj� jd�| _| jj | _ t j | j d�}d|_|j� || _d S )Nr )�Listenerz0starting listener and thread for sending handles)r. )�targetT)r/ r9 r( �AssertionErrorr �debugr r0 r. r2 r) r% ZThread�_serveZdaemon�startr* )r r9 �tr r r r, ~ s z_ResourceSharer._startc C s� t td�r tjtjtdtj�� x�y\| jj� �H}|j� }|d krBP |\}}| j j |�\}}z|||� W d |� X W d Q R X W q" t j � s�t jt j� � Y q"X q"W d S )N�pthread_sigmaskr )�hasattr�signalr@ � SIG_BLOCK�range�NSIGr( ZacceptZrecvr# �popr Z is_exiting�sys� excepthook�exc_info)r r �msgr3 Zdestination_pidr r r r r r= � s z_ResourceSharer._serve)N) r r r r r r � staticmethodr r r+ r, r= r r r r r! = s r! )r rB r rG r% � r �contextr r �__all__�platform�objectr r r! r r r r r r �<module> s ` __pycache__/synchronize.cpython-36.opt-1.pyc 0000644 00000026043 15204243011 0014711 0 ustar 00 3 \/ � @ s, d dddddgZ ddlZddlZddlZddlZddlZdd lmZ dd lmZ dd lm Z ydd lm Z m Z W n e k r� e d ��Y nX e ed��\ZZej jZG dd� de�Z G dd� de �ZG dd� de�ZG dd � d e �ZG dd� de �ZG dd� de�ZG dd� de�ZG dd� dej�ZdS ) �Lock�RLock� Semaphore�BoundedSemaphore� Condition�Event� N� )�context)�process)�util)�SemLock� sem_unlinkz*This platform lacks a functioning sem_openz( implementation, therefore, the requiredz+ synchronization primitives needed will notz function, see issue 3770.� c @ s\ e Zd Zej� Zdd� Zedd� �Zdd� Z dd� Z d d � Z d d � Z d d� Z edd� �ZdS )r c C s� |d krt jj� }|j� }tjdkp*|dk}xPtd�D ]<}y tj|||| j � |� }| _ W n t k rn Y q6X P q6W t d��t j d|j � | j� tjdkr�dd� } t j| | � | j jd k r�dd lm} | | j j� t j| tj| j jfd d � d S ) N�win32�fork�d z cannot find name for semaphorez created semlock with handle %sc S s | j j� d S )N)�_semlock� _after_fork)�obj� r �3/usr/lib64/python3.6/multiprocessing/synchronize.pyr G s z%SemLock.__init__.<locals>._after_forkr )�registerr )Z exitpriority)r Z_default_contextZ get_contextZget_start_method�sys�platform�range�_multiprocessingr � _make_namer �FileExistsErrorr �debug�handle� _make_methodsZregister_after_fork�name�semaphore_trackerr ZFinalize�_cleanup) �self�kind�value�maxvalue�ctxr! Z unlink_now�i�slr r r r r �__init__2 s. zSemLock.__init__c C s ddl m} t| � || � d S )Nr )� unregister)r" r, r )r! r, r r r r# T s zSemLock._cleanupc C s | j j| _| j j| _d S )N)r �acquire�release)r$ r r r r Z s zSemLock._make_methodsc C s | j j� S )N)r � __enter__)r$ r r r r/ ^ s zSemLock.__enter__c G s | j j|� S )N)r �__exit__)r$ �argsr r r r0 a s zSemLock.__exit__c C sD t j| � | j}tjdkr,t j� j|j�}n|j}||j|j |j fS )Nr ) r �assert_spawningr r r Zget_spawning_popenZduplicate_for_childr r% r' r! )r$ r* �hr r r � __getstate__d s zSemLock.__getstate__c C s, t jj|� | _tjd|d � | j� d S )Nz recreated blocker with handle %rr )r r Z_rebuildr r r r )r$ �stater r r � __setstate__m s zSemLock.__setstate__c C s dt j� jd ttj�f S )Nz%s-%sZ semprefix)r �current_processZ_config�nextr �_randr r r r r r s zSemLock._make_nameN)�__name__� __module__� __qualname__�tempfileZ_RandomNameSequencer9 r+ � staticmethodr# r r/ r0 r4 r6 r r r r r r . s " r c @ s& e Zd Zd dd�Zdd� Zdd� ZdS ) r r c C s t j| t|t|d� d S )N)r( )r r+ � SEMAPHORE� SEM_VALUE_MAX)r$ r& r( r r r r+ } s zSemaphore.__init__c C s | j j� S )N)r � _get_value)r$ r r r � get_value� s zSemaphore.get_valuec C s8 y| j j� }W n tk r& d}Y nX d| jj|f S )N�unknownz<%s(value=%s)>)r rA � Exception� __class__r: )r$ r& r r r �__repr__� s zSemaphore.__repr__N)r )r: r; r< r+ rB rF r r r r r { s c @ s e Zd Zddd�Zdd� ZdS )r r c C s t j| t|||d� d S )N)r( )r r+ r? )r$ r& r( r r r r+ � s zBoundedSemaphore.__init__c C s> y| j j� }W n tk r& d}Y nX d| jj|| j jf S )NrC z<%s(value=%s, maxvalue=%s)>)r rA rD rE r: r' )r$ r& r r r rF � s zBoundedSemaphore.__repr__N)r )r: r; r< r+ rF r r r r r � s c @ s e Zd Zdd� Zdd� ZdS )r c C s t j| tdd|d� d S )Nr )r( )r r+ r? )r$ r( r r r r+ � s z Lock.__init__c C s� yf| j j� r8tj� j}tj� jdkrd|dtj� j 7 }n,| j j� dkrLd}n| j j� dkr`d}nd}W n t k r~ d}Y nX d | j j |f S ) N� MainThread�|r �Noner �SomeOtherThread�SomeOtherProcessrC z<%s(owner=%s)>) r �_is_miner r7 r! � threading�current_threadrA �_countrD rE r: )r$ r! r r r rF � s z Lock.__repr__N)r: r; r< r+ rF r r r r r � s c @ s e Zd Zdd� Zdd� ZdS )r c C s t j| tdd|d� d S )Nr )r( )r r+ �RECURSIVE_MUTEX)r$ r( r r r r+ � s zRLock.__init__c C s� y|| j j� rBtj� j}tj� jdkr6|dtj� j 7 }| j j� }n8| j j� dkrZd \}}n | j j� dkrrd \}}nd \}}W n t k r� d\}}Y nX d | j j ||f S )NrG rH r rI r rJ �nonzerorK rC z <%s(%s, %s)>)rI r )rJ rQ )rK rQ )rC rC ) r rL r r7 r! rM rN rO rA rD rE r: )r$ r! �countr r r rF � s zRLock.__repr__N)r: r; r< r+ rF r r r r r � s c @ sj e Zd Zddd�Zdd� Zdd� Zdd � Zd d � Zd d � Zdd� Z ddd�Z dd� Z dd� Z ddd�Z dS )r Nc C s> |p |j � | _|jd�| _|jd�| _|jd�| _| j� d S )Nr )r �_lockr �_sleeping_count� _woken_count�_wait_semaphorer )r$ �lockr( r r r r+ � s zCondition.__init__c C s t j| � | j| j| j| jfS )N)r r2 rS rT rU rV )r$ r r r r4 � s zCondition.__getstate__c C s |\| _ | _| _| _| j� d S )N)rS rT rU rV r )r$ r5 r r r r6 � s zCondition.__setstate__c C s | j j� S )N)rS r/ )r$ r r r r/ � s zCondition.__enter__c G s | j j|� S )N)rS r0 )r$ r1 r r r r0 � s zCondition.__exit__c C s | j j| _| j j| _d S )N)rS r- r. )r$ r r r r � s zCondition._make_methodsc C sJ y | j jj� | jjj� }W n tk r4 d}Y nX d| jj| j|f S )NrC z <%s(%s, %s)>)rT r rA rU rD rE r: rS )r$ Z num_waitersr r r rF � s zCondition.__repr__c C sn | j j� | jjj� }xt|�D ]}| jj� q W z| jjd|�S | jj� xt|�D ]}| jj� qVW X d S )NT) rT r. rS r rO r rV r- rU )r$ �timeoutrR r) r r r �wait� s zCondition.waitc C sN x | j jd�r | jjd�}qW | jjd�rJ| jj� | j j� | jjd� d S )NF)rU r- rT rV r. )r$ �resr r r �notify s zCondition.notifyc C s| x | j jd�r | jjd�}qW d}x"| jjd�rD| jj� |d7 }q$W |rxxt|�D ]}| j j� qTW x| jjd�rvqhW d S )NFr r )rU r- rT rV r. r )r$ rZ Zsleepersr) r r r � notify_all s zCondition.notify_allc C sh |� }|r|S |d k r$t j� | }nd }d }x6|sb|d k rP|t j� }|dkrPP | j|� |� }q.W |S )Nr )�timeZ monotonicrY )r$ Z predicaterX �resultZendtimeZwaittimer r r �wait_for6 s zCondition.wait_for)N)N)N)r: r; r< r+ r4 r6 r/ r0 r rF rY r[ r\ r_ r r r r r � s c @ s6 e Zd Zdd� Zdd� Zdd� Zdd� Zd d d �Zd S ) r c C s |j |j� �| _|jd�| _d S )Nr )r r �_condr �_flag)r$ r( r r r r+ N s zEvent.__init__c C s0 | j � | jjd�r"| jj� dS dS Q R X d S )NFT)r` ra r- r. )r$ r r r �is_setR s z Event.is_setc C s6 | j �& | jjd� | jj� | j j� W d Q R X d S )NF)r` ra r- r. r\ )r$ r r r �setY s z Event.setc C s" | j � | jjd� W d Q R X d S )NF)r` ra r- )r$ r r r �clear_ s z Event.clearNc C sT | j �D | jjd�r | jj� n | j j|� | jjd�rF| jj� dS dS Q R X d S )NFT)r` ra r- r. rY )r$ rX r r r rY c s z Event.wait)N)r: r; r< r+ rb rc rd rY r r r r r L s c @ sZ e Zd Zddd�Zdd� Zdd� Zedd � �Zejd d � �Zed d � �Z e jd d � �Z dS )�BarrierNc C sR dd l }ddlm} ||jd�d �}|j� }| j|||||f� d| _d| _d S )Nr r )� BufferWrapperr) r )�struct�heaprf Zcalcsizer r6 �_staterO ) r$ Zparties�actionrX r( rg rf �wrapperZcondr r r r+ u s zBarrier.__init__c C s. |\| _ | _| _| _| _| jj� jd�| _d S )Nr) )�_parties�_action�_timeoutr` �_wrapperZcreate_memoryview�cast�_array)r$ r5 r r r r6 ~ s zBarrier.__setstate__c C s | j | j| j| j| jfS )N)rl rm rn r` ro )r$ r r r r4 � s zBarrier.__getstate__c C s | j d S )Nr )rq )r$ r r r ri � s zBarrier._statec C s || j d< d S )Nr )rq )r$ r& r r r ri � s c C s | j d S )Nr )rq )r$ r r r rO � s zBarrier._countc C s || j d< d S )Nr )rq )r$ r& r r r rO � s )NN) r: r; r< r+ r6 r4 �propertyri �setterrO r r r r re s s re zRThis platform lacks a functioning sem_open implementation, therefore, the requiredz}This platform lacks a functioning sem_open implementation, therefore, the required synchronization primitives needed will notz�This platform lacks a functioning sem_open implementation, therefore, the required synchronization primitives needed will not function, see issue 3770.)�__all__rM r r= r r] � r r r r r � ImportError�listr rP r? r@ �objectr r r r r r re r r r r �<module> s0 Mz' __pycache__/synchronize.cpython-36.opt-2.pyc 0000644 00000026043 15204243011 0014712 0 ustar 00 3 \/ � @ s, d dddddgZ ddlZddlZddlZddlZddlZdd lmZ dd lmZ dd lm Z ydd lm Z m Z W n e k r� e d ��Y nX e ed��\ZZej jZG dd� de�Z G dd� de �ZG dd� de�ZG dd � d e �ZG dd� de �ZG dd� de�ZG dd� de�ZG dd� dej�ZdS ) �Lock�RLock� Semaphore�BoundedSemaphore� Condition�Event� N� )�context)�process)�util)�SemLock� sem_unlinkz*This platform lacks a functioning sem_openz( implementation, therefore, the requiredz+ synchronization primitives needed will notz function, see issue 3770.� c @ s\ e Zd Zej� Zdd� Zedd� �Zdd� Z dd� Z d d � Z d d � Z d d� Z edd� �ZdS )r c C s� |d krt jj� }|j� }tjdkp*|dk}xPtd�D ]<}y tj|||| j � |� }| _ W n t k rn Y q6X P q6W t d��t j d|j � | j� tjdkr�dd� } t j| | � | j jd k r�dd lm} | | j j� t j| tj| j jfd d � d S ) N�win32�fork�d z cannot find name for semaphorez created semlock with handle %sc S s | j j� d S )N)�_semlock� _after_fork)�obj� r �3/usr/lib64/python3.6/multiprocessing/synchronize.pyr G s z%SemLock.__init__.<locals>._after_forkr )�registerr )Z exitpriority)r Z_default_contextZ get_contextZget_start_method�sys�platform�range�_multiprocessingr � _make_namer �FileExistsErrorr �debug�handle� _make_methodsZregister_after_fork�name�semaphore_trackerr ZFinalize�_cleanup) �self�kind�value�maxvalue�ctxr! Z unlink_now�i�slr r r r r �__init__2 s. zSemLock.__init__c C s ddl m} t| � || � d S )Nr )� unregister)r" r, r )r! r, r r r r# T s zSemLock._cleanupc C s | j j| _| j j| _d S )N)r �acquire�release)r$ r r r r Z s zSemLock._make_methodsc C s | j j� S )N)r � __enter__)r$ r r r r/ ^ s zSemLock.__enter__c G s | j j|� S )N)r �__exit__)r$ �argsr r r r0 a s zSemLock.__exit__c C sD t j| � | j}tjdkr,t j� j|j�}n|j}||j|j |j fS )Nr ) r �assert_spawningr r r Zget_spawning_popenZduplicate_for_childr r% r' r! )r$ r* �hr r r � __getstate__d s zSemLock.__getstate__c C s, t jj|� | _tjd|d � | j� d S )Nz recreated blocker with handle %rr )r r Z_rebuildr r r r )r$ �stater r r � __setstate__m s zSemLock.__setstate__c C s dt j� jd ttj�f S )Nz%s-%sZ semprefix)r �current_processZ_config�nextr �_randr r r r r r s zSemLock._make_nameN)�__name__� __module__� __qualname__�tempfileZ_RandomNameSequencer9 r+ � staticmethodr# r r/ r0 r4 r6 r r r r r r . s " r c @ s& e Zd Zd dd�Zdd� Zdd� ZdS ) r r c C s t j| t|t|d� d S )N)r( )r r+ � SEMAPHORE� SEM_VALUE_MAX)r$ r& r( r r r r+ } s zSemaphore.__init__c C s | j j� S )N)r � _get_value)r$ r r r � get_value� s zSemaphore.get_valuec C s8 y| j j� }W n tk r& d}Y nX d| jj|f S )N�unknownz<%s(value=%s)>)r rA � Exception� __class__r: )r$ r& r r r �__repr__� s zSemaphore.__repr__N)r )r: r; r< r+ rB rF r r r r r { s c @ s e Zd Zddd�Zdd� ZdS )r r c C s t j| t|||d� d S )N)r( )r r+ r? )r$ r& r( r r r r+ � s zBoundedSemaphore.__init__c C s> y| j j� }W n tk r& d}Y nX d| jj|| j jf S )NrC z<%s(value=%s, maxvalue=%s)>)r rA rD rE r: r' )r$ r& r r r rF � s zBoundedSemaphore.__repr__N)r )r: r; r< r+ rF r r r r r � s c @ s e Zd Zdd� Zdd� ZdS )r c C s t j| tdd|d� d S )Nr )r( )r r+ r? )r$ r( r r r r+ � s z Lock.__init__c C s� yf| j j� r8tj� j}tj� jdkrd|dtj� j 7 }n,| j j� dkrLd}n| j j� dkr`d}nd}W n t k r~ d}Y nX d | j j |f S ) N� MainThread�|r �Noner �SomeOtherThread�SomeOtherProcessrC z<%s(owner=%s)>) r �_is_miner r7 r! � threading�current_threadrA �_countrD rE r: )r$ r! r r r rF � s z Lock.__repr__N)r: r; r< r+ rF r r r r r � s c @ s e Zd Zdd� Zdd� ZdS )r c C s t j| tdd|d� d S )Nr )r( )r r+ �RECURSIVE_MUTEX)r$ r( r r r r+ � s zRLock.__init__c C s� y|| j j� rBtj� j}tj� jdkr6|dtj� j 7 }| j j� }n8| j j� dkrZd \}}n | j j� dkrrd \}}nd \}}W n t k r� d\}}Y nX d | j j ||f S )NrG rH r rI r rJ �nonzerorK rC z <%s(%s, %s)>)rI r )rJ rQ )rK rQ )rC rC ) r rL r r7 r! rM rN rO rA rD rE r: )r$ r! �countr r r rF � s zRLock.__repr__N)r: r; r< r+ rF r r r r r � s c @ sj e Zd Zddd�Zdd� Zdd� Zdd � Zd d � Zd d � Zdd� Z ddd�Z dd� Z dd� Z ddd�Z dS )r Nc C s> |p |j � | _|jd�| _|jd�| _|jd�| _| j� d S )Nr )r �_lockr �_sleeping_count� _woken_count�_wait_semaphorer )r$ �lockr( r r r r+ � s zCondition.__init__c C s t j| � | j| j| j| jfS )N)r r2 rS rT rU rV )r$ r r r r4 � s zCondition.__getstate__c C s |\| _ | _| _| _| j� d S )N)rS rT rU rV r )r$ r5 r r r r6 � s zCondition.__setstate__c C s | j j� S )N)rS r/ )r$ r r r r/ � s zCondition.__enter__c G s | j j|� S )N)rS r0 )r$ r1 r r r r0 � s zCondition.__exit__c C s | j j| _| j j| _d S )N)rS r- r. )r$ r r r r � s zCondition._make_methodsc C sJ y | j jj� | jjj� }W n tk r4 d}Y nX d| jj| j|f S )NrC z <%s(%s, %s)>)rT r rA rU rD rE r: rS )r$ Z num_waitersr r r rF � s zCondition.__repr__c C sn | j j� | jjj� }xt|�D ]}| jj� q W z| jjd|�S | jj� xt|�D ]}| jj� qVW X d S )NT) rT r. rS r rO r rV r- rU )r$ �timeoutrR r) r r r �wait� s zCondition.waitc C sN x | j jd�r | jjd�}qW | jjd�rJ| jj� | j j� | jjd� d S )NF)rU r- rT rV r. )r$ �resr r r �notify s zCondition.notifyc C s| x | j jd�r | jjd�}qW d}x"| jjd�rD| jj� |d7 }q$W |rxxt|�D ]}| j j� qTW x| jjd�rvqhW d S )NFr r )rU r- rT rV r. r )r$ rZ Zsleepersr) r r r � notify_all s zCondition.notify_allc C sh |� }|r|S |d k r$t j� | }nd }d }x6|sb|d k rP|t j� }|dkrPP | j|� |� }q.W |S )Nr )�timeZ monotonicrY )r$ Z predicaterX �resultZendtimeZwaittimer r r �wait_for6 s zCondition.wait_for)N)N)N)r: r; r< r+ r4 r6 r/ r0 r rF rY r[ r\ r_ r r r r r � s c @ s6 e Zd Zdd� Zdd� Zdd� Zdd� Zd d d �Zd S ) r c C s |j |j� �| _|jd�| _d S )Nr )r r �_condr �_flag)r$ r( r r r r+ N s zEvent.__init__c C s0 | j � | jjd�r"| jj� dS dS Q R X d S )NFT)r` ra r- r. )r$ r r r �is_setR s z Event.is_setc C s6 | j �& | jjd� | jj� | j j� W d Q R X d S )NF)r` ra r- r. r\ )r$ r r r �setY s z Event.setc C s" | j � | jjd� W d Q R X d S )NF)r` ra r- )r$ r r r �clear_ s z Event.clearNc C sT | j �D | jjd�r | jj� n | j j|� | jjd�rF| jj� dS dS Q R X d S )NFT)r` ra r- r. rY )r$ rX r r r rY c s z Event.wait)N)r: r; r< r+ rb rc rd rY r r r r r L s c @ sZ e Zd Zddd�Zdd� Zdd� Zedd � �Zejd d � �Zed d � �Z e jd d � �Z dS )�BarrierNc C sR dd l }ddlm} ||jd�d �}|j� }| j|||||f� d| _d| _d S )Nr r )� BufferWrapperr) r )�struct�heaprf Zcalcsizer r6 �_staterO ) r$ Zparties�actionrX r( rg rf �wrapperZcondr r r r+ u s zBarrier.__init__c C s. |\| _ | _| _| _| _| jj� jd�| _d S )Nr) )�_parties�_action�_timeoutr` �_wrapperZcreate_memoryview�cast�_array)r$ r5 r r r r6 ~ s zBarrier.__setstate__c C s | j | j| j| j| jfS )N)rl rm rn r` ro )r$ r r r r4 � s zBarrier.__getstate__c C s | j d S )Nr )rq )r$ r r r ri � s zBarrier._statec C s || j d< d S )Nr )rq )r$ r& r r r ri � s c C s | j d S )Nr )rq )r$ r r r rO � s zBarrier._countc C s || j d< d S )Nr )rq )r$ r& r r r rO � s )NN) r: r; r< r+ r6 r4 �propertyri �setterrO r r r r re s s re zRThis platform lacks a functioning sem_open implementation, therefore, the requiredz}This platform lacks a functioning sem_open implementation, therefore, the required synchronization primitives needed will notz�This platform lacks a functioning sem_open implementation, therefore, the required synchronization primitives needed will not function, see issue 3770.)�__all__rM r r= r r] � r r r r r � ImportError�listr rP r? r@ �objectr r r r r r re r r r r �<module> s0 Mz' sharedctypes.py 0000644 00000014145 15204243011 0007611 0 ustar 00 # # Module which supports allocation of ctypes objects from shared memory # # multiprocessing/sharedctypes.py # # Copyright (c) 2006-2008, R Oudkerk # Licensed to PSF under a Contributor Agreement. # import ctypes import weakref from . import heap from . import get_context from .context import reduction, assert_spawning _ForkingPickler = reduction.ForkingPickler __all__ = ['RawValue', 'RawArray', 'Value', 'Array', 'copy', 'synchronized'] # # # typecode_to_type = { 'c': ctypes.c_char, 'u': ctypes.c_wchar, 'b': ctypes.c_byte, 'B': ctypes.c_ubyte, 'h': ctypes.c_short, 'H': ctypes.c_ushort, 'i': ctypes.c_int, 'I': ctypes.c_uint, 'l': ctypes.c_long, 'L': ctypes.c_ulong, 'f': ctypes.c_float, 'd': ctypes.c_double } # # # def _new_value(type_): size = ctypes.sizeof(type_) wrapper = heap.BufferWrapper(size) return rebuild_ctype(type_, wrapper, None) def RawValue(typecode_or_type, *args): ''' Returns a ctypes object allocated from shared memory ''' type_ = typecode_to_type.get(typecode_or_type, typecode_or_type) obj = _new_value(type_) ctypes.memset(ctypes.addressof(obj), 0, ctypes.sizeof(obj)) obj.__init__(*args) return obj def RawArray(typecode_or_type, size_or_initializer): ''' Returns a ctypes array allocated from shared memory ''' type_ = typecode_to_type.get(typecode_or_type, typecode_or_type) if isinstance(size_or_initializer, int): type_ = type_ * size_or_initializer obj = _new_value(type_) ctypes.memset(ctypes.addressof(obj), 0, ctypes.sizeof(obj)) return obj else: type_ = type_ * len(size_or_initializer) result = _new_value(type_) result.__init__(*size_or_initializer) return result def Value(typecode_or_type, *args, lock=True, ctx=None): ''' Return a synchronization wrapper for a Value ''' obj = RawValue(typecode_or_type, *args) if lock is False: return obj if lock in (True, None): ctx = ctx or get_context() lock = ctx.RLock() if not hasattr(lock, 'acquire'): raise AttributeError("'%r' has no method 'acquire'" % lock) return synchronized(obj, lock, ctx=ctx) def Array(typecode_or_type, size_or_initializer, *, lock=True, ctx=None): ''' Return a synchronization wrapper for a RawArray ''' obj = RawArray(typecode_or_type, size_or_initializer) if lock is False: return obj if lock in (True, None): ctx = ctx or get_context() lock = ctx.RLock() if not hasattr(lock, 'acquire'): raise AttributeError("'%r' has no method 'acquire'" % lock) return synchronized(obj, lock, ctx=ctx) def copy(obj): new_obj = _new_value(type(obj)) ctypes.pointer(new_obj)[0] = obj return new_obj def synchronized(obj, lock=None, ctx=None): assert not isinstance(obj, SynchronizedBase), 'object already synchronized' ctx = ctx or get_context() if isinstance(obj, ctypes._SimpleCData): return Synchronized(obj, lock, ctx) elif isinstance(obj, ctypes.Array): if obj._type_ is ctypes.c_char: return SynchronizedString(obj, lock, ctx) return SynchronizedArray(obj, lock, ctx) else: cls = type(obj) try: scls = class_cache[cls] except KeyError: names = [field[0] for field in cls._fields_] d = dict((name, make_property(name)) for name in names) classname = 'Synchronized' + cls.__name__ scls = class_cache[cls] = type(classname, (SynchronizedBase,), d) return scls(obj, lock, ctx) # # Functions for pickling/unpickling # def reduce_ctype(obj): assert_spawning(obj) if isinstance(obj, ctypes.Array): return rebuild_ctype, (obj._type_, obj._wrapper, obj._length_) else: return rebuild_ctype, (type(obj), obj._wrapper, None) def rebuild_ctype(type_, wrapper, length): if length is not None: type_ = type_ * length _ForkingPickler.register(type_, reduce_ctype) buf = wrapper.create_memoryview() obj = type_.from_buffer(buf) obj._wrapper = wrapper return obj # # Function to create properties # def make_property(name): try: return prop_cache[name] except KeyError: d = {} exec(template % ((name,)*7), d) prop_cache[name] = d[name] return d[name] template = ''' def get%s(self): self.acquire() try: return self._obj.%s finally: self.release() def set%s(self, value): self.acquire() try: self._obj.%s = value finally: self.release() %s = property(get%s, set%s) ''' prop_cache = {} class_cache = weakref.WeakKeyDictionary() # # Synchronized wrappers # class SynchronizedBase(object): def __init__(self, obj, lock=None, ctx=None): self._obj = obj if lock: self._lock = lock else: ctx = ctx or get_context(force=True) self._lock = ctx.RLock() self.acquire = self._lock.acquire self.release = self._lock.release def __enter__(self): return self._lock.__enter__() def __exit__(self, *args): return self._lock.__exit__(*args) def __reduce__(self): assert_spawning(self) return synchronized, (self._obj, self._lock) def get_obj(self): return self._obj def get_lock(self): return self._lock def __repr__(self): return '<%s wrapper for %s>' % (type(self).__name__, self._obj) class Synchronized(SynchronizedBase): value = make_property('value') class SynchronizedArray(SynchronizedBase): def __len__(self): return len(self._obj) def __getitem__(self, i): with self: return self._obj[i] def __setitem__(self, i, value): with self: self._obj[i] = value def __getslice__(self, start, stop): with self: return self._obj[start:stop] def __setslice__(self, start, stop, values): with self: self._obj[start:stop] = values class SynchronizedString(SynchronizedArray): value = make_property('value') raw = make_property('raw') popen_spawn_posix.py 0000644 00000003560 15204243011 0010665 0 ustar 00 import io import os from .context import reduction, set_spawning_popen from . import popen_fork from . import spawn from . import util __all__ = ['Popen'] # # Wrapper for an fd used while launching a process # class _DupFd(object): def __init__(self, fd): self.fd = fd def detach(self): return self.fd # # Start child process using a fresh interpreter # class Popen(popen_fork.Popen): method = 'spawn' DupFd = _DupFd def __init__(self, process_obj): self._fds = [] super().__init__(process_obj) def duplicate_for_child(self, fd): self._fds.append(fd) return fd def _launch(self, process_obj): from . import semaphore_tracker tracker_fd = semaphore_tracker.getfd() self._fds.append(tracker_fd) prep_data = spawn.get_preparation_data(process_obj._name) fp = io.BytesIO() set_spawning_popen(self) try: reduction.dump(prep_data, fp) reduction.dump(process_obj, fp) finally: set_spawning_popen(None) parent_r = child_w = child_r = parent_w = None try: parent_r, child_w = os.pipe() child_r, parent_w = os.pipe() cmd = spawn.get_command_line(tracker_fd=tracker_fd, pipe_handle=child_r) self._fds.extend([child_r, child_w]) self.pid = util.spawnv_passfds(spawn.get_executable(), cmd, self._fds) self.sentinel = parent_r with open(parent_w, 'wb', closefd=False) as f: f.write(fp.getbuffer()) finally: if parent_r is not None: util.Finalize(self, os.close, (parent_r,)) for fd in (child_r, child_w, parent_w): if fd is not None: os.close(fd) forkserver.py 0000644 00000020766 15204243011 0007311 0 ustar 00 import errno import os import selectors import signal import socket import struct import sys import threading from . import connection from . import process from .context import reduction from . import semaphore_tracker from . import spawn from . import util __all__ = ['ensure_running', 'get_inherited_fds', 'connect_to_new_process', 'set_forkserver_preload'] # # # MAXFDS_TO_SEND = 256 UNSIGNED_STRUCT = struct.Struct('Q') # large enough for pid_t # # Forkserver class # class ForkServer(object): def __init__(self): self._forkserver_address = None self._forkserver_alive_fd = None self._forkserver_pid = None self._inherited_fds = None self._lock = threading.Lock() self._preload_modules = ['__main__'] def set_forkserver_preload(self, modules_names): '''Set list of module names to try to load in forkserver process.''' if not all(type(mod) is str for mod in self._preload_modules): raise TypeError('module_names must be a list of strings') self._preload_modules = modules_names def get_inherited_fds(self): '''Return list of fds inherited from parent process. This returns None if the current process was not started by fork server. ''' return self._inherited_fds def connect_to_new_process(self, fds): '''Request forkserver to create a child process. Returns a pair of fds (status_r, data_w). The calling process can read the child process's pid and (eventually) its returncode from status_r. The calling process should write to data_w the pickled preparation and process data. ''' self.ensure_running() if len(fds) + 4 >= MAXFDS_TO_SEND: raise ValueError('too many fds') with socket.socket(socket.AF_UNIX) as client: client.connect(self._forkserver_address) parent_r, child_w = os.pipe() child_r, parent_w = os.pipe() allfds = [child_r, child_w, self._forkserver_alive_fd, semaphore_tracker.getfd()] allfds += fds try: reduction.sendfds(client, allfds) return parent_r, parent_w except: os.close(parent_r) os.close(parent_w) raise finally: os.close(child_r) os.close(child_w) def ensure_running(self): '''Make sure that a fork server is running. This can be called from any process. Note that usually a child process will just reuse the forkserver started by its parent, so ensure_running() will do nothing. ''' with self._lock: semaphore_tracker.ensure_running() if self._forkserver_pid is not None: # forkserver was launched before, is it still running? pid, status = os.waitpid(self._forkserver_pid, os.WNOHANG) if not pid: # still alive return # dead, launch it again os.close(self._forkserver_alive_fd) self._forkserver_address = None self._forkserver_alive_fd = None self._forkserver_pid = None cmd = ('from multiprocessing.forkserver import main; ' + 'main(%d, %d, %r, **%r)') if self._preload_modules: desired_keys = {'main_path', 'sys_path'} data = spawn.get_preparation_data('ignore') data = dict((x,y) for (x,y) in data.items() if x in desired_keys) else: data = {} with socket.socket(socket.AF_UNIX) as listener: address = connection.arbitrary_address('AF_UNIX') listener.bind(address) os.chmod(address, 0o600) listener.listen() # all client processes own the write end of the "alive" pipe; # when they all terminate the read end becomes ready. alive_r, alive_w = os.pipe() try: fds_to_pass = [listener.fileno(), alive_r] cmd %= (listener.fileno(), alive_r, self._preload_modules, data) exe = spawn.get_executable() args = [exe] + util._args_from_interpreter_flags() args += ['-c', cmd] pid = util.spawnv_passfds(exe, args, fds_to_pass) except: os.close(alive_w) raise finally: os.close(alive_r) self._forkserver_address = address self._forkserver_alive_fd = alive_w self._forkserver_pid = pid # # # def main(listener_fd, alive_r, preload, main_path=None, sys_path=None): '''Run forkserver.''' if preload: if '__main__' in preload and main_path is not None: process.current_process()._inheriting = True try: spawn.import_main_path(main_path) finally: del process.current_process()._inheriting for modname in preload: try: __import__(modname) except ImportError: pass util._close_stdin() handlers = { # no need to reap zombie processes; signal.SIGCHLD: signal.SIG_IGN, # protect the process from ^C signal.SIGINT: signal.SIG_IGN, } old_handlers = {sig: signal.signal(sig, val) for (sig, val) in handlers.items()} with socket.socket(socket.AF_UNIX, fileno=listener_fd) as listener, \ selectors.DefaultSelector() as selector: _forkserver._forkserver_address = listener.getsockname() selector.register(listener, selectors.EVENT_READ) selector.register(alive_r, selectors.EVENT_READ) while True: try: while True: rfds = [key.fileobj for (key, events) in selector.select()] if rfds: break if alive_r in rfds: # EOF because no more client processes left assert os.read(alive_r, 1) == b'' raise SystemExit assert listener in rfds with listener.accept()[0] as s: code = 1 if os.fork() == 0: try: _serve_one(s, listener, alive_r, old_handlers) except Exception: sys.excepthook(*sys.exc_info()) sys.stderr.flush() finally: os._exit(code) except OSError as e: if e.errno != errno.ECONNABORTED: raise def _serve_one(s, listener, alive_r, handlers): # close unnecessary stuff and reset signal handlers listener.close() os.close(alive_r) for sig, val in handlers.items(): signal.signal(sig, val) # receive fds from parent process fds = reduction.recvfds(s, MAXFDS_TO_SEND + 1) s.close() assert len(fds) <= MAXFDS_TO_SEND (child_r, child_w, _forkserver._forkserver_alive_fd, stfd, *_forkserver._inherited_fds) = fds semaphore_tracker._semaphore_tracker._fd = stfd # send pid to client processes write_unsigned(child_w, os.getpid()) # reseed random number generator if 'random' in sys.modules: import random random.seed() # run process object received over pipe code = spawn._main(child_r) # write the exit code to the pipe write_unsigned(child_w, code) # # Read and write unsigned numbers # def read_unsigned(fd): data = b'' length = UNSIGNED_STRUCT.size while len(data) < length: s = os.read(fd, length - len(data)) if not s: raise EOFError('unexpected EOF') data += s return UNSIGNED_STRUCT.unpack(data)[0] def write_unsigned(fd, n): msg = UNSIGNED_STRUCT.pack(n) while msg: nbytes = os.write(fd, msg) if nbytes == 0: raise RuntimeError('should not get here') msg = msg[nbytes:] # # # _forkserver = ForkServer() ensure_running = _forkserver.ensure_running get_inherited_fds = _forkserver.get_inherited_fds connect_to_new_process = _forkserver.connect_to_new_process set_forkserver_preload = _forkserver.set_forkserver_preload popen_forkserver.py 0000644 00000003644 15204243011 0010506 0 ustar 00 import io import os from .context import reduction, set_spawning_popen if not reduction.HAVE_SEND_HANDLE: raise ImportError('No support for sending fds between processes') from . import forkserver from . import popen_fork from . import spawn from . import util __all__ = ['Popen'] # # Wrapper for an fd used while launching a process # class _DupFd(object): def __init__(self, ind): self.ind = ind def detach(self): return forkserver.get_inherited_fds()[self.ind] # # Start child process using a server process # class Popen(popen_fork.Popen): method = 'forkserver' DupFd = _DupFd def __init__(self, process_obj): self._fds = [] super().__init__(process_obj) def duplicate_for_child(self, fd): self._fds.append(fd) return len(self._fds) - 1 def _launch(self, process_obj): prep_data = spawn.get_preparation_data(process_obj._name) buf = io.BytesIO() set_spawning_popen(self) try: reduction.dump(prep_data, buf) reduction.dump(process_obj, buf) finally: set_spawning_popen(None) self.sentinel, w = forkserver.connect_to_new_process(self._fds) util.Finalize(self, os.close, (self.sentinel,)) with open(w, 'wb', closefd=True) as f: f.write(buf.getbuffer()) self.pid = forkserver.read_unsigned(self.sentinel) def poll(self, flag=os.WNOHANG): if self.returncode is None: from multiprocessing.connection import wait timeout = 0 if flag == os.WNOHANG else None if not wait([self.sentinel], timeout): return None try: self.returncode = forkserver.read_unsigned(self.sentinel) except (OSError, EOFError): # The process ended abnormally perhaps because of a signal self.returncode = 255 return self.returncode reduction.py 0000644 00000022012 15204243011 0007077 0 ustar 00 # # Module which deals with pickling of objects. # # multiprocessing/reduction.py # # Copyright (c) 2006-2008, R Oudkerk # Licensed to PSF under a Contributor Agreement. # from abc import ABCMeta, abstractmethod import copyreg import functools import io import os import pickle import socket import sys from . import context __all__ = ['send_handle', 'recv_handle', 'ForkingPickler', 'register', 'dump'] HAVE_SEND_HANDLE = (sys.platform == 'win32' or (hasattr(socket, 'CMSG_LEN') and hasattr(socket, 'SCM_RIGHTS') and hasattr(socket.socket, 'sendmsg'))) # # Pickler subclass # class ForkingPickler(pickle.Pickler): '''Pickler subclass used by multiprocessing.''' _extra_reducers = {} _copyreg_dispatch_table = copyreg.dispatch_table def __init__(self, *args): super().__init__(*args) self.dispatch_table = self._copyreg_dispatch_table.copy() self.dispatch_table.update(self._extra_reducers) @classmethod def register(cls, type, reduce): '''Register a reduce function for a type.''' cls._extra_reducers[type] = reduce @classmethod def dumps(cls, obj, protocol=None): buf = io.BytesIO() cls(buf, protocol).dump(obj) return buf.getbuffer() loads = pickle.loads register = ForkingPickler.register def dump(obj, file, protocol=None): '''Replacement for pickle.dump() using ForkingPickler.''' ForkingPickler(file, protocol).dump(obj) # # Platform specific definitions # if sys.platform == 'win32': # Windows __all__ += ['DupHandle', 'duplicate', 'steal_handle'] import _winapi def duplicate(handle, target_process=None, inheritable=False): '''Duplicate a handle. (target_process is a handle not a pid!)''' if target_process is None: target_process = _winapi.GetCurrentProcess() return _winapi.DuplicateHandle( _winapi.GetCurrentProcess(), handle, target_process, 0, inheritable, _winapi.DUPLICATE_SAME_ACCESS) def steal_handle(source_pid, handle): '''Steal a handle from process identified by source_pid.''' source_process_handle = _winapi.OpenProcess( _winapi.PROCESS_DUP_HANDLE, False, source_pid) try: return _winapi.DuplicateHandle( source_process_handle, handle, _winapi.GetCurrentProcess(), 0, False, _winapi.DUPLICATE_SAME_ACCESS | _winapi.DUPLICATE_CLOSE_SOURCE) finally: _winapi.CloseHandle(source_process_handle) def send_handle(conn, handle, destination_pid): '''Send a handle over a local connection.''' dh = DupHandle(handle, _winapi.DUPLICATE_SAME_ACCESS, destination_pid) conn.send(dh) def recv_handle(conn): '''Receive a handle over a local connection.''' return conn.recv().detach() class DupHandle(object): '''Picklable wrapper for a handle.''' def __init__(self, handle, access, pid=None): if pid is None: # We just duplicate the handle in the current process and # let the receiving process steal the handle. pid = os.getpid() proc = _winapi.OpenProcess(_winapi.PROCESS_DUP_HANDLE, False, pid) try: self._handle = _winapi.DuplicateHandle( _winapi.GetCurrentProcess(), handle, proc, access, False, 0) finally: _winapi.CloseHandle(proc) self._access = access self._pid = pid def detach(self): '''Get the handle. This should only be called once.''' # retrieve handle from process which currently owns it if self._pid == os.getpid(): # The handle has already been duplicated for this process. return self._handle # We must steal the handle from the process whose pid is self._pid. proc = _winapi.OpenProcess(_winapi.PROCESS_DUP_HANDLE, False, self._pid) try: return _winapi.DuplicateHandle( proc, self._handle, _winapi.GetCurrentProcess(), self._access, False, _winapi.DUPLICATE_CLOSE_SOURCE) finally: _winapi.CloseHandle(proc) else: # Unix __all__ += ['DupFd', 'sendfds', 'recvfds'] import array # On MacOSX we should acknowledge receipt of fds -- see Issue14669 ACKNOWLEDGE = sys.platform == 'darwin' def sendfds(sock, fds): '''Send an array of fds over an AF_UNIX socket.''' fds = array.array('i', fds) msg = bytes([len(fds) % 256]) sock.sendmsg([msg], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, fds)]) if ACKNOWLEDGE and sock.recv(1) != b'A': raise RuntimeError('did not receive acknowledgement of fd') def recvfds(sock, size): '''Receive an array of fds over an AF_UNIX socket.''' a = array.array('i') bytes_size = a.itemsize * size msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_SPACE(bytes_size)) if not msg and not ancdata: raise EOFError try: if ACKNOWLEDGE: sock.send(b'A') if len(ancdata) != 1: raise RuntimeError('received %d items of ancdata' % len(ancdata)) cmsg_level, cmsg_type, cmsg_data = ancdata[0] if (cmsg_level == socket.SOL_SOCKET and cmsg_type == socket.SCM_RIGHTS): if len(cmsg_data) % a.itemsize != 0: raise ValueError a.frombytes(cmsg_data) assert len(a) % 256 == msg[0] return list(a) except (ValueError, IndexError): pass raise RuntimeError('Invalid data received') def send_handle(conn, handle, destination_pid): '''Send a handle over a local connection.''' with socket.fromfd(conn.fileno(), socket.AF_UNIX, socket.SOCK_STREAM) as s: sendfds(s, [handle]) def recv_handle(conn): '''Receive a handle over a local connection.''' with socket.fromfd(conn.fileno(), socket.AF_UNIX, socket.SOCK_STREAM) as s: return recvfds(s, 1)[0] def DupFd(fd): '''Return a wrapper for an fd.''' popen_obj = context.get_spawning_popen() if popen_obj is not None: return popen_obj.DupFd(popen_obj.duplicate_for_child(fd)) elif HAVE_SEND_HANDLE: from . import resource_sharer return resource_sharer.DupFd(fd) else: raise ValueError('SCM_RIGHTS appears not to be available') # # Try making some callable types picklable # def _reduce_method(m): if m.__self__ is None: return getattr, (m.__class__, m.__func__.__name__) else: return getattr, (m.__self__, m.__func__.__name__) class _C: def f(self): pass register(type(_C().f), _reduce_method) def _reduce_method_descriptor(m): return getattr, (m.__objclass__, m.__name__) register(type(list.append), _reduce_method_descriptor) register(type(int.__add__), _reduce_method_descriptor) def _reduce_partial(p): return _rebuild_partial, (p.func, p.args, p.keywords or {}) def _rebuild_partial(func, args, keywords): return functools.partial(func, *args, **keywords) register(functools.partial, _reduce_partial) # # Make sockets picklable # if sys.platform == 'win32': def _reduce_socket(s): from .resource_sharer import DupSocket return _rebuild_socket, (DupSocket(s),) def _rebuild_socket(ds): return ds.detach() register(socket.socket, _reduce_socket) else: def _reduce_socket(s): df = DupFd(s.fileno()) return _rebuild_socket, (df, s.family, s.type, s.proto) def _rebuild_socket(df, family, type, proto): fd = df.detach() return socket.socket(family, type, proto, fileno=fd) register(socket.socket, _reduce_socket) class AbstractReducer(metaclass=ABCMeta): '''Abstract base class for use in implementing a Reduction class suitable for use in replacing the standard reduction mechanism used in multiprocessing.''' ForkingPickler = ForkingPickler register = register dump = dump send_handle = send_handle recv_handle = recv_handle if sys.platform == 'win32': steal_handle = steal_handle duplicate = duplicate DupHandle = DupHandle else: sendfds = sendfds recvfds = recvfds DupFd = DupFd _reduce_method = _reduce_method _reduce_method_descriptor = _reduce_method_descriptor _rebuild_partial = _rebuild_partial _reduce_socket = _reduce_socket _rebuild_socket = _rebuild_socket def __init__(self, *args): register(type(_C().f), _reduce_method) register(type(list.append), _reduce_method_descriptor) register(type(int.__add__), _reduce_method_descriptor) register(functools.partial, _reduce_partial) register(socket.socket, _reduce_socket) popen_spawn_win32.py 0000644 00000005667 15204243012 0010500 0 ustar 00 import os import msvcrt import signal import sys import _winapi from .context import reduction, get_spawning_popen, set_spawning_popen from . import spawn from . import util __all__ = ['Popen'] # # # TERMINATE = 0x10000 WINEXE = (sys.platform == 'win32' and getattr(sys, 'frozen', False)) WINSERVICE = sys.executable.lower().endswith("pythonservice.exe") # # We define a Popen class similar to the one from subprocess, but # whose constructor takes a process object as its argument. # class Popen(object): ''' Start a subprocess to run the code of a process object ''' method = 'spawn' def __init__(self, process_obj): prep_data = spawn.get_preparation_data(process_obj._name) # read end of pipe will be "stolen" by the child process # -- see spawn_main() in spawn.py. rhandle, whandle = _winapi.CreatePipe(None, 0) wfd = msvcrt.open_osfhandle(whandle, 0) cmd = spawn.get_command_line(parent_pid=os.getpid(), pipe_handle=rhandle) cmd = ' '.join('"%s"' % x for x in cmd) with open(wfd, 'wb', closefd=True) as to_child: # start process try: hp, ht, pid, tid = _winapi.CreateProcess( spawn.get_executable(), cmd, None, None, False, 0, None, None, None) _winapi.CloseHandle(ht) except: _winapi.CloseHandle(rhandle) raise # set attributes of self self.pid = pid self.returncode = None self._handle = hp self.sentinel = int(hp) util.Finalize(self, _winapi.CloseHandle, (self.sentinel,)) # send information to child set_spawning_popen(self) try: reduction.dump(prep_data, to_child) reduction.dump(process_obj, to_child) finally: set_spawning_popen(None) def duplicate_for_child(self, handle): assert self is get_spawning_popen() return reduction.duplicate(handle, self.sentinel) def wait(self, timeout=None): if self.returncode is None: if timeout is None: msecs = _winapi.INFINITE else: msecs = max(0, int(timeout * 1000 + 0.5)) res = _winapi.WaitForSingleObject(int(self._handle), msecs) if res == _winapi.WAIT_OBJECT_0: code = _winapi.GetExitCodeProcess(self._handle) if code == TERMINATE: code = -signal.SIGTERM self.returncode = code return self.returncode def poll(self): return self.wait(timeout=0) def terminate(self): if self.returncode is None: try: _winapi.TerminateProcess(int(self._handle), TERMINATE) except OSError: if self.wait(timeout=1.0) is None: raise queues.py 0000644 00000025013 15204243012 0006417 0 ustar 00 # # Module implementing queues # # multiprocessing/queues.py # # Copyright (c) 2006-2008, R Oudkerk # Licensed to PSF under a Contributor Agreement. # __all__ = ['Queue', 'SimpleQueue', 'JoinableQueue'] import sys import os import threading import collections import time import weakref import errno from queue import Empty, Full import _multiprocessing from . import connection from . import context _ForkingPickler = context.reduction.ForkingPickler from .util import debug, info, Finalize, register_after_fork, is_exiting # # Queue type using a pipe, buffer and thread # class Queue(object): def __init__(self, maxsize=0, *, ctx): if maxsize <= 0: # Can raise ImportError (see issues #3770 and #23400) from .synchronize import SEM_VALUE_MAX as maxsize self._maxsize = maxsize self._reader, self._writer = connection.Pipe(duplex=False) self._rlock = ctx.Lock() self._opid = os.getpid() if sys.platform == 'win32': self._wlock = None else: self._wlock = ctx.Lock() self._sem = ctx.BoundedSemaphore(maxsize) # For use by concurrent.futures self._ignore_epipe = False self._after_fork() if sys.platform != 'win32': register_after_fork(self, Queue._after_fork) def __getstate__(self): context.assert_spawning(self) return (self._ignore_epipe, self._maxsize, self._reader, self._writer, self._rlock, self._wlock, self._sem, self._opid) def __setstate__(self, state): (self._ignore_epipe, self._maxsize, self._reader, self._writer, self._rlock, self._wlock, self._sem, self._opid) = state self._after_fork() def _after_fork(self): debug('Queue._after_fork()') self._notempty = threading.Condition(threading.Lock()) self._buffer = collections.deque() self._thread = None self._jointhread = None self._joincancelled = False self._closed = False self._close = None self._send_bytes = self._writer.send_bytes self._recv_bytes = self._reader.recv_bytes self._poll = self._reader.poll def put(self, obj, block=True, timeout=None): assert not self._closed if not self._sem.acquire(block, timeout): raise Full with self._notempty: if self._thread is None: self._start_thread() self._buffer.append(obj) self._notempty.notify() def get(self, block=True, timeout=None): if block and timeout is None: with self._rlock: res = self._recv_bytes() self._sem.release() else: if block: deadline = time.monotonic() + timeout if not self._rlock.acquire(block, timeout): raise Empty try: if block: timeout = deadline - time.monotonic() if not self._poll(timeout): raise Empty elif not self._poll(): raise Empty res = self._recv_bytes() self._sem.release() finally: self._rlock.release() # unserialize the data after having released the lock return _ForkingPickler.loads(res) def qsize(self): # Raises NotImplementedError on Mac OSX because of broken sem_getvalue() return self._maxsize - self._sem._semlock._get_value() def empty(self): return not self._poll() def full(self): return self._sem._semlock._is_zero() def get_nowait(self): return self.get(False) def put_nowait(self, obj): return self.put(obj, False) def close(self): self._closed = True try: self._reader.close() finally: close = self._close if close: self._close = None close() def join_thread(self): debug('Queue.join_thread()') assert self._closed if self._jointhread: self._jointhread() def cancel_join_thread(self): debug('Queue.cancel_join_thread()') self._joincancelled = True try: self._jointhread.cancel() except AttributeError: pass def _start_thread(self): debug('Queue._start_thread()') # Start thread which transfers data from buffer to pipe self._buffer.clear() self._thread = threading.Thread( target=Queue._feed, args=(self._buffer, self._notempty, self._send_bytes, self._wlock, self._writer.close, self._ignore_epipe), name='QueueFeederThread' ) self._thread.daemon = True debug('doing self._thread.start()') self._thread.start() debug('... done self._thread.start()') if not self._joincancelled: self._jointhread = Finalize( self._thread, Queue._finalize_join, [weakref.ref(self._thread)], exitpriority=-5 ) # Send sentinel to the thread queue object when garbage collected self._close = Finalize( self, Queue._finalize_close, [self._buffer, self._notempty], exitpriority=10 ) @staticmethod def _finalize_join(twr): debug('joining queue thread') thread = twr() if thread is not None: thread.join() debug('... queue thread joined') else: debug('... queue thread already dead') @staticmethod def _finalize_close(buffer, notempty): debug('telling queue thread to quit') with notempty: buffer.append(_sentinel) notempty.notify() @staticmethod def _feed(buffer, notempty, send_bytes, writelock, close, ignore_epipe): debug('starting thread to feed data to pipe') nacquire = notempty.acquire nrelease = notempty.release nwait = notempty.wait bpopleft = buffer.popleft sentinel = _sentinel if sys.platform != 'win32': wacquire = writelock.acquire wrelease = writelock.release else: wacquire = None while 1: try: nacquire() try: if not buffer: nwait() finally: nrelease() try: while 1: obj = bpopleft() if obj is sentinel: debug('feeder thread got sentinel -- exiting') close() return # serialize the data before acquiring the lock obj = _ForkingPickler.dumps(obj) if wacquire is None: send_bytes(obj) else: wacquire() try: send_bytes(obj) finally: wrelease() except IndexError: pass except Exception as e: if ignore_epipe and getattr(e, 'errno', 0) == errno.EPIPE: return # Since this runs in a daemon thread the resources it uses # may be become unusable while the process is cleaning up. # We ignore errors which happen after the process has # started to cleanup. if is_exiting(): info('error in queue thread: %s', e) return else: import traceback traceback.print_exc() _sentinel = object() # # A queue type which also supports join() and task_done() methods # # Note that if you do not call task_done() for each finished task then # eventually the counter's semaphore may overflow causing Bad Things # to happen. # class JoinableQueue(Queue): def __init__(self, maxsize=0, *, ctx): Queue.__init__(self, maxsize, ctx=ctx) self._unfinished_tasks = ctx.Semaphore(0) self._cond = ctx.Condition() def __getstate__(self): return Queue.__getstate__(self) + (self._cond, self._unfinished_tasks) def __setstate__(self, state): Queue.__setstate__(self, state[:-2]) self._cond, self._unfinished_tasks = state[-2:] def put(self, obj, block=True, timeout=None): assert not self._closed if not self._sem.acquire(block, timeout): raise Full with self._notempty, self._cond: if self._thread is None: self._start_thread() self._buffer.append(obj) self._unfinished_tasks.release() self._notempty.notify() def task_done(self): with self._cond: if not self._unfinished_tasks.acquire(False): raise ValueError('task_done() called too many times') if self._unfinished_tasks._semlock._is_zero(): self._cond.notify_all() def join(self): with self._cond: if not self._unfinished_tasks._semlock._is_zero(): self._cond.wait() # # Simplified Queue type -- really just a locked pipe # class SimpleQueue(object): def __init__(self, *, ctx): self._reader, self._writer = connection.Pipe(duplex=False) self._rlock = ctx.Lock() self._poll = self._reader.poll if sys.platform == 'win32': self._wlock = None else: self._wlock = ctx.Lock() def empty(self): return not self._poll() def __getstate__(self): context.assert_spawning(self) return (self._reader, self._writer, self._rlock, self._wlock) def __setstate__(self, state): (self._reader, self._writer, self._rlock, self._wlock) = state self._poll = self._reader.poll def get(self): with self._rlock: res = self._reader.recv_bytes() # unserialize the data after having released the lock return _ForkingPickler.loads(res) def put(self, obj): # serialize the data before acquiring the lock obj = _ForkingPickler.dumps(obj) if self._wlock is None: # writes to a message oriented win32 pipe are atomic self._writer.send_bytes(obj) else: with self._wlock: self._writer.send_bytes(obj) dummy/__init__.py 0000644 00000005520 15204243012 0010003 0 ustar 00 # # Support for the API of the multiprocessing package using threads # # multiprocessing/dummy/__init__.py # # Copyright (c) 2006-2008, R Oudkerk # Licensed to PSF under a Contributor Agreement. # __all__ = [ 'Process', 'current_process', 'active_children', 'freeze_support', 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Condition', 'Event', 'Barrier', 'Queue', 'Manager', 'Pipe', 'Pool', 'JoinableQueue' ] # # Imports # import threading import sys import weakref import array from .connection import Pipe from threading import Lock, RLock, Semaphore, BoundedSemaphore from threading import Event, Condition, Barrier from queue import Queue # # # class DummyProcess(threading.Thread): def __init__(self, group=None, target=None, name=None, args=(), kwargs={}): threading.Thread.__init__(self, group, target, name, args, kwargs) self._pid = None self._children = weakref.WeakKeyDictionary() self._start_called = False self._parent = current_process() def start(self): assert self._parent is current_process() self._start_called = True if hasattr(self._parent, '_children'): self._parent._children[self] = None threading.Thread.start(self) @property def exitcode(self): if self._start_called and not self.is_alive(): return 0 else: return None # # # Process = DummyProcess current_process = threading.current_thread current_process()._children = weakref.WeakKeyDictionary() def active_children(): children = current_process()._children for p in list(children): if not p.is_alive(): children.pop(p, None) return list(children) def freeze_support(): pass # # # class Namespace(object): def __init__(self, **kwds): self.__dict__.update(kwds) def __repr__(self): items = list(self.__dict__.items()) temp = [] for name, value in items: if not name.startswith('_'): temp.append('%s=%r' % (name, value)) temp.sort() return '%s(%s)' % (self.__class__.__name__, ', '.join(temp)) dict = dict list = list def Array(typecode, sequence, lock=True): return array.array(typecode, sequence) class Value(object): def __init__(self, typecode, value, lock=True): self._typecode = typecode self._value = value def _get(self): return self._value def _set(self, value): self._value = value value = property(_get, _set) def __repr__(self): return '<%s(%r, %r)>'%(type(self).__name__,self._typecode,self._value) def Manager(): return sys.modules[__name__] def shutdown(): pass def Pool(processes=None, initializer=None, initargs=()): from ..pool import ThreadPool return ThreadPool(processes, initializer, initargs) JoinableQueue = Queue dummy/connection.py 0000644 00000003057 15204243012 0010406 0 ustar 00 # # Analogue of `multiprocessing.connection` which uses queues instead of sockets # # multiprocessing/dummy/connection.py # # Copyright (c) 2006-2008, R Oudkerk # Licensed to PSF under a Contributor Agreement. # __all__ = [ 'Client', 'Listener', 'Pipe' ] from queue import Queue families = [None] class Listener(object): def __init__(self, address=None, family=None, backlog=1): self._backlog_queue = Queue(backlog) def accept(self): return Connection(*self._backlog_queue.get()) def close(self): self._backlog_queue = None address = property(lambda self: self._backlog_queue) def __enter__(self): return self def __exit__(self, exc_type, exc_value, exc_tb): self.close() def Client(address): _in, _out = Queue(), Queue() address.put((_out, _in)) return Connection(_in, _out) def Pipe(duplex=True): a, b = Queue(), Queue() return Connection(a, b), Connection(b, a) class Connection(object): def __init__(self, _in, _out): self._out = _out self._in = _in self.send = self.send_bytes = _out.put self.recv = self.recv_bytes = _in.get def poll(self, timeout=0.0): if self._in.qsize() > 0: return True if timeout <= 0.0: return False with self._in.not_empty: self._in.not_empty.wait(timeout) return self._in.qsize() > 0 def close(self): pass def __enter__(self): return self def __exit__(self, exc_type, exc_value, exc_tb): self.close() dummy/__pycache__/__init__.cpython-36.opt-2.pyc 0000644 00000007136 15204243012 0015234 0 ustar 00 3 \P � @ s d ddddddddd d d d d ddgZ ddlZddlZddlZddlZddlmZ ddlmZmZm Z m Z ddlm Z m Z m Z ddlmZ G dd� dej�ZeZejZej� e� _dd� Zdd� ZG dd � d e�ZeZe Z d&d d �Z G d d!� d!e�Z d"d � Z d#d$� Z ddf fd%d�Z!eZ"dS )'�Process�current_process�active_children�freeze_support�Lock�RLock� Semaphore�BoundedSemaphore� Condition�Event�Barrier�Queue�Manager�Pipe�Pool� JoinableQueue� N� )r )r r r r )r r r )r c @ s4 e Zd Zdddf i fdd�Zdd� Zedd� �ZdS )� DummyProcessNc C s8 t jj| |||||� d | _tj� | _d| _t� | _ d S )NF) � threading�Thread�__init__Z_pid�weakref�WeakKeyDictionary� _children� _start_calledr �_parent)�self�group�target�name�args�kwargs� r" �6/usr/lib64/python3.6/multiprocessing/dummy/__init__.pyr $ s zDummyProcess.__init__c C s. d| _ t| jd�r d | jj| < tjj| � d S )NTr )r �hasattrr r r r �start)r r" r" r# r% + s zDummyProcess.startc C s | j r| j� rdS d S d S )Nr )r �is_alive)r r" r" r# �exitcode2 s zDummyProcess.exitcode)�__name__� __module__� __qualname__r r% �propertyr' r" r" r" r# r " s r c C s6 t � j} x$t| �D ]}|j� s| j|d � qW t| �S )N)r r �listr& �pop)Zchildren�pr" r" r# r A s c C s d S )Nr" r" r" r" r# r H s c @ s e Zd Zdd� Zdd� ZdS )� Namespacec K s | j j|� d S )N)�__dict__�update)r �kwdsr" r" r# r P s zNamespace.__init__c C s^ t | jj� �}g }x,|D ]$\}}|jd�s|jd||f � qW |j� d| jjdj|�f S )N�_z%s=%rz%s(%s)z, ) r, r0 �items� startswith�append�sort� __class__r( �join)r r4 Ztempr �valuer" r" r# �__repr__R s zNamespace.__repr__N)r( r) r* r r; r" r" r" r# r/ O s r/ Tc C s t j | |�S )N)�array)�typecodeZsequence�lockr" r" r# �Array^ s r? c @ s8 e Zd Zd dd�Zdd� Zdd� Zeee�Zdd � Zd S ) �ValueTc C s || _ || _d S )N)� _typecode�_value)r r= r: r> r" r" r# r b s zValue.__init__c C s | j S )N)rB )r r" r" r# �_gete s z Value._getc C s || _ d S )N)rB )r r: r" r" r# �_setg s z Value._setc C s dt | �j| j| jf S )Nz <%s(%r, %r)>)�typer( rA rB )r r" r" r# r; j s zValue.__repr__N)T) r( r) r* r rC rD r+ r: r; r" r" r" r# r@ a s r@ c C s t jt S )N)�sys�modulesr( r" r" r" r# r m s c C s d S )Nr" r" r" r" r# �shutdownp s rH c C s ddl m} || ||�S )N� )� ThreadPool)ZpoolrJ )Z processesZ initializerZinitargsrJ r" r" r# r s s )T)#�__all__r rF r r< Z connectionr r r r r r r r Zqueuer r r r Zcurrent_threadr r r r r �objectr/ �dictr, r? r@ r rH r r r" r" r" r# �<module> s2 dummy/__pycache__/connection.cpython-36.opt-2.pyc 0000644 00000004701 15204243012 0015627 0 ustar 00 3 \/ � @ sR d ddgZ ddlmZ dgZG dd� de�Zdd � Zd d d�ZG d d � d e�ZdS ) �Client�Listener�Pipe� )�QueueNc @ sB e Zd Zddd�Zdd� Zdd� Zed d � �Zd d � Zd d� Z dS )r N� c C s t |�| _d S )N)r �_backlog_queue)�self�addressZfamilyZbacklog� r �8/usr/lib64/python3.6/multiprocessing/dummy/connection.py�__init__ s zListener.__init__c C s t | jj� � S )N)� Connectionr �get)r r r r �accept s zListener.acceptc C s d | _ d S )N)r )r r r r �close s zListener.closec C s | j S )N)r )r r r r �<lambda> s zListener.<lambda>c C s | S )Nr )r r r r � __enter__ s zListener.__enter__c C s | j � d S )N)r )r �exc_type� exc_value�exc_tbr r r �__exit__" s zListener.__exit__)NNr ) �__name__� __module__� __qualname__r r r �propertyr r r r r r r r s c C s& t � t � }}| j||f� t||�S )N)r �putr )r �_in�_outr r r r & s Tc C s" t � t � }}t||�t||�fS )N)r r )Zduplex�a�br r r r , s c @ s6 e Zd Zdd� Zd dd�Zdd� Zdd � Zd d � Zd S )r c C s, || _ || _|j | _| _|j | _| _d S )N)r r r �sendZ send_bytesr ZrecvZ recv_bytes)r r r r r r r 3 s zConnection.__init__� c C sN | j j� dkrdS |dkr dS | j j� | j jj|� W d Q R X | j j� dkS )Nr Tg F)r ZqsizeZ not_empty�wait)r Ztimeoutr r r �poll9 s zConnection.pollc C s d S )Nr )r r r r r B s zConnection.closec C s | S )Nr )r r r r r E s zConnection.__enter__c C s | j � d S )N)r )r r r r r r r r H s zConnection.__exit__N)r! )r r r r r# r r r r r r r r 1 s r )T) �__all__Zqueuer Zfamilies�objectr r r r r r r r �<module> s dummy/__pycache__/connection.cpython-36.opt-1.pyc 0000644 00000004701 15204243012 0015626 0 ustar 00 3 \/ � @ sR d ddgZ ddlmZ dgZG dd� de�Zdd � Zd d d�ZG d d � d e�ZdS ) �Client�Listener�Pipe� )�QueueNc @ sB e Zd Zddd�Zdd� Zdd� Zed d � �Zd d � Zd d� Z dS )r N� c C s t |�| _d S )N)r �_backlog_queue)�self�addressZfamilyZbacklog� r �8/usr/lib64/python3.6/multiprocessing/dummy/connection.py�__init__ s zListener.__init__c C s t | jj� � S )N)� Connectionr �get)r r r r �accept s zListener.acceptc C s d | _ d S )N)r )r r r r �close s zListener.closec C s | j S )N)r )r r r r �<lambda> s zListener.<lambda>c C s | S )Nr )r r r r � __enter__ s zListener.__enter__c C s | j � d S )N)r )r �exc_type� exc_value�exc_tbr r r �__exit__" s zListener.__exit__)NNr ) �__name__� __module__� __qualname__r r r �propertyr r r r r r r r s c C s& t � t � }}| j||f� t||�S )N)r �putr )r �_in�_outr r r r & s Tc C s" t � t � }}t||�t||�fS )N)r r )Zduplex�a�br r r r , s c @ s6 e Zd Zdd� Zd dd�Zdd� Zdd � Zd d � Zd S )r c C s, || _ || _|j | _| _|j | _| _d S )N)r r r �sendZ send_bytesr ZrecvZ recv_bytes)r r r r r r r 3 s zConnection.__init__� c C sN | j j� dkrdS |dkr dS | j j� | j jj|� W d Q R X | j j� dkS )Nr Tg F)r ZqsizeZ not_empty�wait)r Ztimeoutr r r �poll9 s zConnection.pollc C s d S )Nr )r r r r r B s zConnection.closec C s | S )Nr )r r r r r E s zConnection.__enter__c C s | j � d S )N)r )r r r r r r r r H s zConnection.__exit__N)r! )r r r r r# r r r r r r r r 1 s r )T) �__all__Zqueuer Zfamilies�objectr r r r r r r r �<module> s dummy/__pycache__/connection.cpython-36.pyc 0000644 00000004701 15204243012 0014667 0 ustar 00 3 \/ � @ sR d ddgZ ddlmZ dgZG dd� de�Zdd � Zd d d�ZG d d � d e�ZdS ) �Client�Listener�Pipe� )�QueueNc @ sB e Zd Zddd�Zdd� Zdd� Zed d � �Zd d � Zd d� Z dS )r N� c C s t |�| _d S )N)r �_backlog_queue)�self�addressZfamilyZbacklog� r �8/usr/lib64/python3.6/multiprocessing/dummy/connection.py�__init__ s zListener.__init__c C s t | jj� � S )N)� Connectionr �get)r r r r �accept s zListener.acceptc C s d | _ d S )N)r )r r r r �close s zListener.closec C s | j S )N)r )r r r r �<lambda> s zListener.<lambda>c C s | S )Nr )r r r r � __enter__ s zListener.__enter__c C s | j � d S )N)r )r �exc_type� exc_value�exc_tbr r r �__exit__" s zListener.__exit__)NNr ) �__name__� __module__� __qualname__r r r �propertyr r r r r r r r s c C s& t � t � }}| j||f� t||�S )N)r �putr )r �_in�_outr r r r & s Tc C s" t � t � }}t||�t||�fS )N)r r )Zduplex�a�br r r r , s c @ s6 e Zd Zdd� Zd dd�Zdd� Zdd � Zd d � Zd S )r c C s, || _ || _|j | _| _|j | _| _d S )N)r r r �sendZ send_bytesr ZrecvZ recv_bytes)r r r r r r r 3 s zConnection.__init__� c C sN | j j� dkrdS |dkr dS | j j� | j jj|� W d Q R X | j j� dkS )Nr Tg F)r ZqsizeZ not_empty�wait)r Ztimeoutr r r �poll9 s zConnection.pollc C s d S )Nr )r r r r r B s zConnection.closec C s | S )Nr )r r r r r E s zConnection.__enter__c C s | j � d S )N)r )r r r r r r r r H s zConnection.__exit__N)r! )r r r r r# r r r r r r r r 1 s r )T) �__all__Zqueuer Zfamilies�objectr r r r r r r r �<module> s dummy/__pycache__/__init__.cpython-36.pyc 0000644 00000007205 15204243012 0014271 0 ustar 00 3 \P � @ s d ddddddddd d d d d ddgZ ddlZddlZddlZddlZddlmZ ddlmZmZm Z m Z ddlm Z m Z m Z ddlmZ G dd� dej�ZeZejZej� e� _dd� Zdd� ZG dd � d e�ZeZe Z d&d d �Z G d d!� d!e�Z d"d � Z d#d$� Z ddf fd%d�Z!eZ"dS )'�Process�current_process�active_children�freeze_support�Lock�RLock� Semaphore�BoundedSemaphore� Condition�Event�Barrier�Queue�Manager�Pipe�Pool� JoinableQueue� N� )r )r r r r )r r r )r c @ s4 e Zd Zdddf i fdd�Zdd� Zedd� �ZdS )� DummyProcessNc C s8 t jj| |||||� d | _tj� | _d| _t� | _ d S )NF) � threading�Thread�__init__Z_pid�weakref�WeakKeyDictionary� _children� _start_calledr �_parent)�self�group�target�name�args�kwargs� r" �6/usr/lib64/python3.6/multiprocessing/dummy/__init__.pyr $ s zDummyProcess.__init__c C s> | j t� kst�d| _t| j d�r.d | j j| < tjj| � d S )NTr ) r r �AssertionErrorr �hasattrr r r �start)r r" r" r# r& + s zDummyProcess.startc C s | j r| j� rdS d S d S )Nr )r �is_alive)r r" r" r# �exitcode2 s zDummyProcess.exitcode)�__name__� __module__� __qualname__r r& �propertyr( r" r" r" r# r " s r c C s6 t � j} x$t| �D ]}|j� s| j|d � qW t| �S )N)r r �listr' �pop)Zchildren�pr" r" r# r A s c C s d S )Nr" r" r" r" r# r H s c @ s e Zd Zdd� Zdd� ZdS )� Namespacec K s | j j|� d S )N)�__dict__�update)r �kwdsr" r" r# r P s zNamespace.__init__c C s^ t | jj� �}g }x,|D ]$\}}|jd�s|jd||f � qW |j� d| jjdj|�f S )N�_z%s=%rz%s(%s)z, ) r- r1 �items� startswith�append�sort� __class__r) �join)r r5 Ztempr �valuer" r" r# �__repr__R s zNamespace.__repr__N)r) r* r+ r r< r" r" r" r# r0 O s r0 Tc C s t j | |�S )N)�array)�typecodeZsequence�lockr" r" r# �Array^ s r@ c @ s8 e Zd Zd dd�Zdd� Zdd� Zeee�Zdd � Zd S ) �ValueTc C s || _ || _d S )N)� _typecode�_value)r r> r; r? r" r" r# r b s zValue.__init__c C s | j S )N)rC )r r" r" r# �_gete s z Value._getc C s || _ d S )N)rC )r r; r" r" r# �_setg s z Value._setc C s dt | �j| j| jf S )Nz <%s(%r, %r)>)�typer) rB rC )r r" r" r# r< j s zValue.__repr__N)T) r) r* r+ r rD rE r, r; r< r" r" r" r# rA a s rA c C s t jt S )N)�sys�modulesr) r" r" r" r# r m s c C s d S )Nr" r" r" r" r# �shutdownp s rI c C s ddl m} || ||�S )N� )� ThreadPool)ZpoolrK )Z processesZ initializerZinitargsrK r" r" r# r s s )T)#�__all__r rG r r= Z connectionr r r r r r r r Zqueuer r r r Zcurrent_threadr r r r r �objectr0 �dictr- r@ rA r rI r r r" r" r" r# �<module> s2 dummy/__pycache__/__init__.cpython-36.opt-1.pyc 0000644 00000007136 15204243012 0015233 0 ustar 00 3 \P � @ s d ddddddddd d d d d ddgZ ddlZddlZddlZddlZddlmZ ddlmZmZm Z m Z ddlm Z m Z m Z ddlmZ G dd� dej�ZeZejZej� e� _dd� Zdd� ZG dd � d e�ZeZe Z d&d d �Z G d d!� d!e�Z d"d � Z d#d$� Z ddf fd%d�Z!eZ"dS )'�Process�current_process�active_children�freeze_support�Lock�RLock� Semaphore�BoundedSemaphore� Condition�Event�Barrier�Queue�Manager�Pipe�Pool� JoinableQueue� N� )r )r r r r )r r r )r c @ s4 e Zd Zdddf i fdd�Zdd� Zedd� �ZdS )� DummyProcessNc C s8 t jj| |||||� d | _tj� | _d| _t� | _ d S )NF) � threading�Thread�__init__Z_pid�weakref�WeakKeyDictionary� _children� _start_calledr �_parent)�self�group�target�name�args�kwargs� r" �6/usr/lib64/python3.6/multiprocessing/dummy/__init__.pyr $ s zDummyProcess.__init__c C s. d| _ t| jd�r d | jj| < tjj| � d S )NTr )r �hasattrr r r r �start)r r" r" r# r% + s zDummyProcess.startc C s | j r| j� rdS d S d S )Nr )r �is_alive)r r" r" r# �exitcode2 s zDummyProcess.exitcode)�__name__� __module__� __qualname__r r% �propertyr' r" r" r" r# r " s r c C s6 t � j} x$t| �D ]}|j� s| j|d � qW t| �S )N)r r �listr& �pop)Zchildren�pr" r" r# r A s c C s d S )Nr" r" r" r" r# r H s c @ s e Zd Zdd� Zdd� ZdS )� Namespacec K s | j j|� d S )N)�__dict__�update)r �kwdsr" r" r# r P s zNamespace.__init__c C s^ t | jj� �}g }x,|D ]$\}}|jd�s|jd||f � qW |j� d| jjdj|�f S )N�_z%s=%rz%s(%s)z, ) r, r0 �items� startswith�append�sort� __class__r( �join)r r4 Ztempr �valuer" r" r# �__repr__R s zNamespace.__repr__N)r( r) r* r r; r" r" r" r# r/ O s r/ Tc C s t j | |�S )N)�array)�typecodeZsequence�lockr" r" r# �Array^ s r? c @ s8 e Zd Zd dd�Zdd� Zdd� Zeee�Zdd � Zd S ) �ValueTc C s || _ || _d S )N)� _typecode�_value)r r= r: r> r" r" r# r b s zValue.__init__c C s | j S )N)rB )r r" r" r# �_gete s z Value._getc C s || _ d S )N)rB )r r: r" r" r# �_setg s z Value._setc C s dt | �j| j| jf S )Nz <%s(%r, %r)>)�typer( rA rB )r r" r" r# r; j s zValue.__repr__N)T) r( r) r* r rC rD r+ r: r; r" r" r" r# r@ a s r@ c C s t jt S )N)�sys�modulesr( r" r" r" r# r m s c C s d S )Nr" r" r" r" r# �shutdownp s rH c C s ddl m} || ||�S )N� )� ThreadPool)ZpoolrJ )Z processesZ initializerZinitargsrJ r" r" r# r s s )T)#�__all__r rF r r< Z connectionr r r r r r r r Zqueuer r r r Zcurrent_threadr r r r r �objectr/ �dictr, r? r@ r rH r r r" r" r" r# �<module> s2 semaphore_tracker.py 0000644 00000012422 15204243012 0010606 0 ustar 00 # # On Unix we run a server process which keeps track of unlinked # semaphores. The server ignores SIGINT and SIGTERM and reads from a # pipe. Every other process of the program has a copy of the writable # end of the pipe, so we get EOF when all other processes have exited. # Then the server process unlinks any remaining semaphore names. # # This is important because the system only supports a limited number # of named semaphores, and they will not be automatically removed till # the next reboot. Without this semaphore tracker process, "killall # python" would probably leave unlinked semaphores. # import os import signal import sys import threading import warnings import _multiprocessing from . import spawn from . import util __all__ = ['ensure_running', 'register', 'unregister'] class SemaphoreTracker(object): def __init__(self): self._lock = threading.Lock() self._fd = None self._pid = None def getfd(self): self.ensure_running() return self._fd def ensure_running(self): '''Make sure that semaphore tracker process is running. This can be run from any process. Usually a child process will use the semaphore created by its parent.''' with self._lock: if self._pid is not None: # semaphore tracker was launched before, is it still running? pid, status = os.waitpid(self._pid, os.WNOHANG) if not pid: # => still alive return # => dead, launch it again os.close(self._fd) self._fd = None self._pid = None warnings.warn('semaphore_tracker: process died unexpectedly, ' 'relaunching. Some semaphores might leak.') fds_to_pass = [] try: fds_to_pass.append(sys.stderr.fileno()) except Exception: pass cmd = 'from multiprocessing.semaphore_tracker import main;main(%d)' r, w = os.pipe() try: fds_to_pass.append(r) # process will out live us, so no need to wait on pid exe = spawn.get_executable() args = [exe] + util._args_from_interpreter_flags() args += ['-c', cmd % r] pid = util.spawnv_passfds(exe, args, fds_to_pass) except: os.close(w) raise else: self._fd = w self._pid = pid finally: os.close(r) def register(self, name): '''Register name of semaphore with semaphore tracker.''' self._send('REGISTER', name) def unregister(self, name): '''Unregister name of semaphore with semaphore tracker.''' self._send('UNREGISTER', name) def _send(self, cmd, name): self.ensure_running() msg = '{0}:{1}\n'.format(cmd, name).encode('ascii') if len(name) > 512: # posix guarantees that writes to a pipe of less than PIPE_BUF # bytes are atomic, and that PIPE_BUF >= 512 raise ValueError('name too long') nbytes = os.write(self._fd, msg) assert nbytes == len(msg) _semaphore_tracker = SemaphoreTracker() ensure_running = _semaphore_tracker.ensure_running register = _semaphore_tracker.register unregister = _semaphore_tracker.unregister getfd = _semaphore_tracker.getfd def main(fd): '''Run semaphore tracker.''' # protect the process from ^C and "killall python" etc signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGTERM, signal.SIG_IGN) for f in (sys.stdin, sys.stdout): try: f.close() except Exception: pass cache = set() try: # keep track of registered/unregistered semaphores with open(fd, 'rb') as f: for line in f: try: cmd, name = line.strip().split(b':') if cmd == b'REGISTER': cache.add(name) elif cmd == b'UNREGISTER': cache.remove(name) else: raise RuntimeError('unrecognized command %r' % cmd) except Exception: try: sys.excepthook(*sys.exc_info()) except: pass finally: # all processes have terminated; cleanup any remaining semaphores if cache: try: warnings.warn('semaphore_tracker: There appear to be %d ' 'leaked semaphores to clean up at shutdown' % len(cache)) except Exception: pass for name in cache: # For some reason the process which created and registered this # semaphore has failed to unregister it. Presumably it has died. # We therefore unlink it. try: name = name.decode('ascii') try: _multiprocessing.sem_unlink(name) except Exception as e: warnings.warn('semaphore_tracker: %r: %s' % (name, e)) finally: pass resource_sharer.py 0000644 00000012315 15204243012 0010304 0 ustar 00 # # We use a background thread for sharing fds on Unix, and for sharing sockets on # Windows. # # A client which wants to pickle a resource registers it with the resource # sharer and gets an identifier in return. The unpickling process will connect # to the resource sharer, sends the identifier and its pid, and then receives # the resource. # import os import signal import socket import sys import threading from . import process from .context import reduction from . import util __all__ = ['stop'] if sys.platform == 'win32': __all__ += ['DupSocket'] class DupSocket(object): '''Picklable wrapper for a socket.''' def __init__(self, sock): new_sock = sock.dup() def send(conn, pid): share = new_sock.share(pid) conn.send_bytes(share) self._id = _resource_sharer.register(send, new_sock.close) def detach(self): '''Get the socket. This should only be called once.''' with _resource_sharer.get_connection(self._id) as conn: share = conn.recv_bytes() return socket.fromshare(share) else: __all__ += ['DupFd'] class DupFd(object): '''Wrapper for fd which can be used at any time.''' def __init__(self, fd): new_fd = os.dup(fd) def send(conn, pid): reduction.send_handle(conn, new_fd, pid) def close(): os.close(new_fd) self._id = _resource_sharer.register(send, close) def detach(self): '''Get the fd. This should only be called once.''' with _resource_sharer.get_connection(self._id) as conn: return reduction.recv_handle(conn) class _ResourceSharer(object): '''Manager for resouces using background thread.''' def __init__(self): self._key = 0 self._cache = {} self._old_locks = [] self._lock = threading.Lock() self._listener = None self._address = None self._thread = None util.register_after_fork(self, _ResourceSharer._afterfork) def register(self, send, close): '''Register resource, returning an identifier.''' with self._lock: if self._address is None: self._start() self._key += 1 self._cache[self._key] = (send, close) return (self._address, self._key) @staticmethod def get_connection(ident): '''Return connection from which to receive identified resource.''' from .connection import Client address, key = ident c = Client(address, authkey=process.current_process().authkey) c.send((key, os.getpid())) return c def stop(self, timeout=None): '''Stop the background thread and clear registered resources.''' from .connection import Client with self._lock: if self._address is not None: c = Client(self._address, authkey=process.current_process().authkey) c.send(None) c.close() self._thread.join(timeout) if self._thread.is_alive(): util.sub_warning('_ResourceSharer thread did ' 'not stop when asked') self._listener.close() self._thread = None self._address = None self._listener = None for key, (send, close) in self._cache.items(): close() self._cache.clear() def _afterfork(self): for key, (send, close) in self._cache.items(): close() self._cache.clear() # If self._lock was locked at the time of the fork, it may be broken # -- see issue 6721. Replace it without letting it be gc'ed. self._old_locks.append(self._lock) self._lock = threading.Lock() if self._listener is not None: self._listener.close() self._listener = None self._address = None self._thread = None def _start(self): from .connection import Listener assert self._listener is None util.debug('starting listener and thread for sending handles') self._listener = Listener(authkey=process.current_process().authkey) self._address = self._listener.address t = threading.Thread(target=self._serve) t.daemon = True t.start() self._thread = t def _serve(self): if hasattr(signal, 'pthread_sigmask'): signal.pthread_sigmask(signal.SIG_BLOCK, range(1, signal.NSIG)) while 1: try: with self._listener.accept() as conn: msg = conn.recv() if msg is None: break key, destination_pid = msg send, close = self._cache.pop(key) try: send(conn, destination_pid) finally: close() except: if not util.is_exiting(): sys.excepthook(*sys.exc_info()) _resource_sharer = _ResourceSharer() stop = _resource_sharer.stop synchronize.py 0000644 00000027422 15204243012 0007471 0 ustar 00 # # Module implementing synchronization primitives # # multiprocessing/synchronize.py # # Copyright (c) 2006-2008, R Oudkerk # Licensed to PSF under a Contributor Agreement. # __all__ = [ 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Condition', 'Event' ] import threading import sys import tempfile import _multiprocessing import time from . import context from . import process from . import util # Try to import the mp.synchronize module cleanly, if it fails # raise ImportError for platforms lacking a working sem_open implementation. # See issue 3770 try: from _multiprocessing import SemLock, sem_unlink except (ImportError): raise ImportError("This platform lacks a functioning sem_open" + " implementation, therefore, the required" + " synchronization primitives needed will not" + " function, see issue 3770.") # # Constants # RECURSIVE_MUTEX, SEMAPHORE = list(range(2)) SEM_VALUE_MAX = _multiprocessing.SemLock.SEM_VALUE_MAX # # Base class for semaphores and mutexes; wraps `_multiprocessing.SemLock` # class SemLock(object): _rand = tempfile._RandomNameSequence() def __init__(self, kind, value, maxvalue, *, ctx): if ctx is None: ctx = context._default_context.get_context() name = ctx.get_start_method() unlink_now = sys.platform == 'win32' or name == 'fork' for i in range(100): try: sl = self._semlock = _multiprocessing.SemLock( kind, value, maxvalue, self._make_name(), unlink_now) except FileExistsError: pass else: break else: raise FileExistsError('cannot find name for semaphore') util.debug('created semlock with handle %s' % sl.handle) self._make_methods() if sys.platform != 'win32': def _after_fork(obj): obj._semlock._after_fork() util.register_after_fork(self, _after_fork) if self._semlock.name is not None: # We only get here if we are on Unix with forking # disabled. When the object is garbage collected or the # process shuts down we unlink the semaphore name from .semaphore_tracker import register register(self._semlock.name) util.Finalize(self, SemLock._cleanup, (self._semlock.name,), exitpriority=0) @staticmethod def _cleanup(name): from .semaphore_tracker import unregister sem_unlink(name) unregister(name) def _make_methods(self): self.acquire = self._semlock.acquire self.release = self._semlock.release def __enter__(self): return self._semlock.__enter__() def __exit__(self, *args): return self._semlock.__exit__(*args) def __getstate__(self): context.assert_spawning(self) sl = self._semlock if sys.platform == 'win32': h = context.get_spawning_popen().duplicate_for_child(sl.handle) else: h = sl.handle return (h, sl.kind, sl.maxvalue, sl.name) def __setstate__(self, state): self._semlock = _multiprocessing.SemLock._rebuild(*state) util.debug('recreated blocker with handle %r' % state[0]) self._make_methods() @staticmethod def _make_name(): return '%s-%s' % (process.current_process()._config['semprefix'], next(SemLock._rand)) # # Semaphore # class Semaphore(SemLock): def __init__(self, value=1, *, ctx): SemLock.__init__(self, SEMAPHORE, value, SEM_VALUE_MAX, ctx=ctx) def get_value(self): return self._semlock._get_value() def __repr__(self): try: value = self._semlock._get_value() except Exception: value = 'unknown' return '<%s(value=%s)>' % (self.__class__.__name__, value) # # Bounded semaphore # class BoundedSemaphore(Semaphore): def __init__(self, value=1, *, ctx): SemLock.__init__(self, SEMAPHORE, value, value, ctx=ctx) def __repr__(self): try: value = self._semlock._get_value() except Exception: value = 'unknown' return '<%s(value=%s, maxvalue=%s)>' % \ (self.__class__.__name__, value, self._semlock.maxvalue) # # Non-recursive lock # class Lock(SemLock): def __init__(self, *, ctx): SemLock.__init__(self, SEMAPHORE, 1, 1, ctx=ctx) def __repr__(self): try: if self._semlock._is_mine(): name = process.current_process().name if threading.current_thread().name != 'MainThread': name += '|' + threading.current_thread().name elif self._semlock._get_value() == 1: name = 'None' elif self._semlock._count() > 0: name = 'SomeOtherThread' else: name = 'SomeOtherProcess' except Exception: name = 'unknown' return '<%s(owner=%s)>' % (self.__class__.__name__, name) # # Recursive lock # class RLock(SemLock): def __init__(self, *, ctx): SemLock.__init__(self, RECURSIVE_MUTEX, 1, 1, ctx=ctx) def __repr__(self): try: if self._semlock._is_mine(): name = process.current_process().name if threading.current_thread().name != 'MainThread': name += '|' + threading.current_thread().name count = self._semlock._count() elif self._semlock._get_value() == 1: name, count = 'None', 0 elif self._semlock._count() > 0: name, count = 'SomeOtherThread', 'nonzero' else: name, count = 'SomeOtherProcess', 'nonzero' except Exception: name, count = 'unknown', 'unknown' return '<%s(%s, %s)>' % (self.__class__.__name__, name, count) # # Condition variable # class Condition(object): def __init__(self, lock=None, *, ctx): self._lock = lock or ctx.RLock() self._sleeping_count = ctx.Semaphore(0) self._woken_count = ctx.Semaphore(0) self._wait_semaphore = ctx.Semaphore(0) self._make_methods() def __getstate__(self): context.assert_spawning(self) return (self._lock, self._sleeping_count, self._woken_count, self._wait_semaphore) def __setstate__(self, state): (self._lock, self._sleeping_count, self._woken_count, self._wait_semaphore) = state self._make_methods() def __enter__(self): return self._lock.__enter__() def __exit__(self, *args): return self._lock.__exit__(*args) def _make_methods(self): self.acquire = self._lock.acquire self.release = self._lock.release def __repr__(self): try: num_waiters = (self._sleeping_count._semlock._get_value() - self._woken_count._semlock._get_value()) except Exception: num_waiters = 'unknown' return '<%s(%s, %s)>' % (self.__class__.__name__, self._lock, num_waiters) def wait(self, timeout=None): assert self._lock._semlock._is_mine(), \ 'must acquire() condition before using wait()' # indicate that this thread is going to sleep self._sleeping_count.release() # release lock count = self._lock._semlock._count() for i in range(count): self._lock.release() try: # wait for notification or timeout return self._wait_semaphore.acquire(True, timeout) finally: # indicate that this thread has woken self._woken_count.release() # reacquire lock for i in range(count): self._lock.acquire() def notify(self): assert self._lock._semlock._is_mine(), 'lock is not owned' assert not self._wait_semaphore.acquire(False) # to take account of timeouts since last notify() we subtract # woken_count from sleeping_count and rezero woken_count while self._woken_count.acquire(False): res = self._sleeping_count.acquire(False) assert res if self._sleeping_count.acquire(False): # try grabbing a sleeper self._wait_semaphore.release() # wake up one sleeper self._woken_count.acquire() # wait for the sleeper to wake # rezero _wait_semaphore in case a timeout just happened self._wait_semaphore.acquire(False) def notify_all(self): assert self._lock._semlock._is_mine(), 'lock is not owned' assert not self._wait_semaphore.acquire(False) # to take account of timeouts since last notify*() we subtract # woken_count from sleeping_count and rezero woken_count while self._woken_count.acquire(False): res = self._sleeping_count.acquire(False) assert res sleepers = 0 while self._sleeping_count.acquire(False): self._wait_semaphore.release() # wake up one sleeper sleepers += 1 if sleepers: for i in range(sleepers): self._woken_count.acquire() # wait for a sleeper to wake # rezero wait_semaphore in case some timeouts just happened while self._wait_semaphore.acquire(False): pass def wait_for(self, predicate, timeout=None): result = predicate() if result: return result if timeout is not None: endtime = time.monotonic() + timeout else: endtime = None waittime = None while not result: if endtime is not None: waittime = endtime - time.monotonic() if waittime <= 0: break self.wait(waittime) result = predicate() return result # # Event # class Event(object): def __init__(self, *, ctx): self._cond = ctx.Condition(ctx.Lock()) self._flag = ctx.Semaphore(0) def is_set(self): with self._cond: if self._flag.acquire(False): self._flag.release() return True return False def set(self): with self._cond: self._flag.acquire(False) self._flag.release() self._cond.notify_all() def clear(self): with self._cond: self._flag.acquire(False) def wait(self, timeout=None): with self._cond: if self._flag.acquire(False): self._flag.release() else: self._cond.wait(timeout) if self._flag.acquire(False): self._flag.release() return True return False # # Barrier # class Barrier(threading.Barrier): def __init__(self, parties, action=None, timeout=None, *, ctx): import struct from .heap import BufferWrapper wrapper = BufferWrapper(struct.calcsize('i') * 2) cond = ctx.Condition() self.__setstate__((parties, action, timeout, cond, wrapper)) self._state = 0 self._count = 0 def __setstate__(self, state): (self._parties, self._action, self._timeout, self._cond, self._wrapper) = state self._array = self._wrapper.create_memoryview().cast('i') def __getstate__(self): return (self._parties, self._action, self._timeout, self._cond, self._wrapper) @property def _state(self): return self._array[0] @_state.setter def _state(self, value): self._array[0] = value @property def _count(self): return self._array[1] @_count.setter def _count(self, value): self._array[1] = value process.py 0000644 00000021773 15204243012 0006577 0 ustar 00 # # Module providing the `Process` class which emulates `threading.Thread` # # multiprocessing/process.py # # Copyright (c) 2006-2008, R Oudkerk # Licensed to PSF under a Contributor Agreement. # __all__ = ['BaseProcess', 'current_process', 'active_children'] # # Imports # import os import sys import signal import itertools from _weakrefset import WeakSet # # # try: ORIGINAL_DIR = os.path.abspath(os.getcwd()) except OSError: ORIGINAL_DIR = None # # Public functions # def current_process(): ''' Return process object representing the current process ''' return _current_process def active_children(): ''' Return list of process objects corresponding to live child processes ''' _cleanup() return list(_children) # # # def _cleanup(): # check for processes which have finished for p in list(_children): if p._popen.poll() is not None: _children.discard(p) # # The `Process` class # class BaseProcess(object): ''' Process objects represent activity that is run in a separate process The class is analogous to `threading.Thread` ''' def _Popen(self): raise NotImplementedError def __init__(self, group=None, target=None, name=None, args=(), kwargs={}, *, daemon=None): assert group is None, 'group argument must be None for now' count = next(_process_counter) self._identity = _current_process._identity + (count,) self._config = _current_process._config.copy() self._parent_pid = os.getpid() self._popen = None self._target = target self._args = tuple(args) self._kwargs = dict(kwargs) self._name = name or type(self).__name__ + '-' + \ ':'.join(str(i) for i in self._identity) if daemon is not None: self.daemon = daemon _dangling.add(self) def run(self): ''' Method to be run in sub-process; can be overridden in sub-class ''' if self._target: self._target(*self._args, **self._kwargs) def start(self): ''' Start child process ''' assert self._popen is None, 'cannot start a process twice' assert self._parent_pid == os.getpid(), \ 'can only start a process object created by current process' assert not _current_process._config.get('daemon'), \ 'daemonic processes are not allowed to have children' _cleanup() self._popen = self._Popen(self) self._sentinel = self._popen.sentinel # Avoid a refcycle if the target function holds an indirect # reference to the process object (see bpo-30775) del self._target, self._args, self._kwargs _children.add(self) def terminate(self): ''' Terminate process; sends SIGTERM signal or uses TerminateProcess() ''' self._popen.terminate() def join(self, timeout=None): ''' Wait until child process terminates ''' assert self._parent_pid == os.getpid(), 'can only join a child process' assert self._popen is not None, 'can only join a started process' res = self._popen.wait(timeout) if res is not None: _children.discard(self) def is_alive(self): ''' Return whether process is alive ''' if self is _current_process: return True assert self._parent_pid == os.getpid(), 'can only test a child process' if self._popen is None: return False returncode = self._popen.poll() if returncode is None: return True else: _children.discard(self) return False @property def name(self): return self._name @name.setter def name(self, name): assert isinstance(name, str), 'name must be a string' self._name = name @property def daemon(self): ''' Return whether process is a daemon ''' return self._config.get('daemon', False) @daemon.setter def daemon(self, daemonic): ''' Set whether process is a daemon ''' assert self._popen is None, 'process has already started' self._config['daemon'] = daemonic @property def authkey(self): return self._config['authkey'] @authkey.setter def authkey(self, authkey): ''' Set authorization key of process ''' self._config['authkey'] = AuthenticationString(authkey) @property def exitcode(self): ''' Return exit code of process or `None` if it has yet to stop ''' if self._popen is None: return self._popen return self._popen.poll() @property def ident(self): ''' Return identifier (PID) of process or `None` if it has yet to start ''' if self is _current_process: return os.getpid() else: return self._popen and self._popen.pid pid = ident @property def sentinel(self): ''' Return a file descriptor (Unix) or handle (Windows) suitable for waiting for process termination. ''' try: return self._sentinel except AttributeError: raise ValueError("process not started") def __repr__(self): if self is _current_process: status = 'started' elif self._parent_pid != os.getpid(): status = 'unknown' elif self._popen is None: status = 'initial' else: if self._popen.poll() is not None: status = self.exitcode else: status = 'started' if type(status) is int: if status == 0: status = 'stopped' else: status = 'stopped[%s]' % _exitcode_to_name.get(status, status) return '<%s(%s, %s%s)>' % (type(self).__name__, self._name, status, self.daemon and ' daemon' or '') ## def _bootstrap(self): from . import util, context global _current_process, _process_counter, _children try: if self._start_method is not None: context._force_start_method(self._start_method) _process_counter = itertools.count(1) _children = set() util._close_stdin() old_process = _current_process _current_process = self try: util._finalizer_registry.clear() util._run_after_forkers() finally: # delay finalization of the old process object until after # _run_after_forkers() is executed del old_process util.info('child process calling self.run()') try: self.run() exitcode = 0 finally: util._exit_function() except SystemExit as e: if not e.args: exitcode = 1 elif isinstance(e.args[0], int): exitcode = e.args[0] else: sys.stderr.write(str(e.args[0]) + '\n') exitcode = 1 except: exitcode = 1 import traceback sys.stderr.write('Process %s:\n' % self.name) traceback.print_exc() finally: util.info('process exiting with exitcode %d' % exitcode) util._flush_std_streams() return exitcode # # We subclass bytes to avoid accidental transmission of auth keys over network # class AuthenticationString(bytes): def __reduce__(self): from .context import get_spawning_popen if get_spawning_popen() is None: raise TypeError( 'Pickling an AuthenticationString object is ' 'disallowed for security reasons' ) return AuthenticationString, (bytes(self),) # # Create object representing the main process # class _MainProcess(BaseProcess): def __init__(self): self._identity = () self._name = 'MainProcess' self._parent_pid = None self._popen = None self._config = {'authkey': AuthenticationString(os.urandom(32)), 'semprefix': '/mp'} # Note that some versions of FreeBSD only allow named # semaphores to have names of up to 14 characters. Therefore # we choose a short prefix. # # On MacOSX in a sandbox it may be necessary to use a # different prefix -- see #19478. # # Everything in self._config will be inherited by descendant # processes. _current_process = _MainProcess() _process_counter = itertools.count(1) _children = set() del _MainProcess # # Give names to some return codes # _exitcode_to_name = {} for name, signum in list(signal.__dict__.items()): if name[:3]=='SIG' and '_' not in name: _exitcode_to_name[-signum] = name # For debug and leak testing _dangling = WeakSet() spawn.py 0000644 00000021237 15204243012 0006244 0 ustar 00 # # Code used to start processes when using the spawn or forkserver # start methods. # # multiprocessing/spawn.py # # Copyright (c) 2006-2008, R Oudkerk # Licensed to PSF under a Contributor Agreement. # import os import sys import runpy import types from . import get_start_method, set_start_method from . import process from .context import reduction from . import util __all__ = ['_main', 'freeze_support', 'set_executable', 'get_executable', 'get_preparation_data', 'get_command_line', 'import_main_path'] # # _python_exe is the assumed path to the python executable. # People embedding Python want to modify it. # if sys.platform != 'win32': WINEXE = False WINSERVICE = False else: WINEXE = (sys.platform == 'win32' and getattr(sys, 'frozen', False)) WINSERVICE = sys.executable.lower().endswith("pythonservice.exe") if WINSERVICE: _python_exe = os.path.join(sys.exec_prefix, 'python.exe') else: _python_exe = sys.executable def set_executable(exe): global _python_exe _python_exe = exe def get_executable(): return _python_exe # # # def is_forking(argv): ''' Return whether commandline indicates we are forking ''' if len(argv) >= 2 and argv[1] == '--multiprocessing-fork': return True else: return False def freeze_support(): ''' Run code for process object if this in not the main process ''' if is_forking(sys.argv): kwds = {} for arg in sys.argv[2:]: name, value = arg.split('=') if value == 'None': kwds[name] = None else: kwds[name] = int(value) spawn_main(**kwds) sys.exit() def get_command_line(**kwds): ''' Returns prefix of command line used for spawning a child process ''' if getattr(sys, 'frozen', False): return ([sys.executable, '--multiprocessing-fork'] + ['%s=%r' % item for item in kwds.items()]) else: prog = 'from multiprocessing.spawn import spawn_main; spawn_main(%s)' prog %= ', '.join('%s=%r' % item for item in kwds.items()) opts = util._args_from_interpreter_flags() return [_python_exe] + opts + ['-c', prog, '--multiprocessing-fork'] def spawn_main(pipe_handle, parent_pid=None, tracker_fd=None): ''' Run code specified by data received over pipe ''' assert is_forking(sys.argv) if sys.platform == 'win32': import msvcrt new_handle = reduction.steal_handle(parent_pid, pipe_handle) fd = msvcrt.open_osfhandle(new_handle, os.O_RDONLY) else: from . import semaphore_tracker semaphore_tracker._semaphore_tracker._fd = tracker_fd fd = pipe_handle exitcode = _main(fd) sys.exit(exitcode) def _main(fd): with os.fdopen(fd, 'rb', closefd=True) as from_parent: process.current_process()._inheriting = True try: preparation_data = reduction.pickle.load(from_parent) prepare(preparation_data) self = reduction.pickle.load(from_parent) finally: del process.current_process()._inheriting return self._bootstrap() def _check_not_importing_main(): if getattr(process.current_process(), '_inheriting', False): raise RuntimeError(''' An attempt has been made to start a new process before the current process has finished its bootstrapping phase. This probably means that you are not using fork to start your child processes and you have forgotten to use the proper idiom in the main module: if __name__ == '__main__': freeze_support() ... The "freeze_support()" line can be omitted if the program is not going to be frozen to produce an executable.''') def get_preparation_data(name): ''' Return info about parent needed by child to unpickle process object ''' _check_not_importing_main() d = dict( log_to_stderr=util._log_to_stderr, authkey=process.current_process().authkey, ) if util._logger is not None: d['log_level'] = util._logger.getEffectiveLevel() sys_path=sys.path.copy() try: i = sys_path.index('') except ValueError: pass else: sys_path[i] = process.ORIGINAL_DIR d.update( name=name, sys_path=sys_path, sys_argv=sys.argv, orig_dir=process.ORIGINAL_DIR, dir=os.getcwd(), start_method=get_start_method(), ) # Figure out whether to initialise main in the subprocess as a module # or through direct execution (or to leave it alone entirely) main_module = sys.modules['__main__'] main_mod_name = getattr(main_module.__spec__, "name", None) if main_mod_name is not None: d['init_main_from_name'] = main_mod_name elif sys.platform != 'win32' or (not WINEXE and not WINSERVICE): main_path = getattr(main_module, '__file__', None) if main_path is not None: if (not os.path.isabs(main_path) and process.ORIGINAL_DIR is not None): main_path = os.path.join(process.ORIGINAL_DIR, main_path) d['init_main_from_path'] = os.path.normpath(main_path) return d # # Prepare current process # old_main_modules = [] def prepare(data): ''' Try to get current process ready to unpickle process object ''' if 'name' in data: process.current_process().name = data['name'] if 'authkey' in data: process.current_process().authkey = data['authkey'] if 'log_to_stderr' in data and data['log_to_stderr']: util.log_to_stderr() if 'log_level' in data: util.get_logger().setLevel(data['log_level']) if 'sys_path' in data: sys.path = data['sys_path'] if 'sys_argv' in data: sys.argv = data['sys_argv'] if 'dir' in data: os.chdir(data['dir']) if 'orig_dir' in data: process.ORIGINAL_DIR = data['orig_dir'] if 'start_method' in data: set_start_method(data['start_method'], force=True) if 'init_main_from_name' in data: _fixup_main_from_name(data['init_main_from_name']) elif 'init_main_from_path' in data: _fixup_main_from_path(data['init_main_from_path']) # Multiprocessing module helpers to fix up the main module in # spawned subprocesses def _fixup_main_from_name(mod_name): # __main__.py files for packages, directories, zip archives, etc, run # their "main only" code unconditionally, so we don't even try to # populate anything in __main__, nor do we make any changes to # __main__ attributes current_main = sys.modules['__main__'] if mod_name == "__main__" or mod_name.endswith(".__main__"): return # If this process was forked, __main__ may already be populated if getattr(current_main.__spec__, "name", None) == mod_name: return # Otherwise, __main__ may contain some non-main code where we need to # support unpickling it properly. We rerun it as __mp_main__ and make # the normal __main__ an alias to that old_main_modules.append(current_main) main_module = types.ModuleType("__mp_main__") main_content = runpy.run_module(mod_name, run_name="__mp_main__", alter_sys=True) main_module.__dict__.update(main_content) sys.modules['__main__'] = sys.modules['__mp_main__'] = main_module def _fixup_main_from_path(main_path): # If this process was forked, __main__ may already be populated current_main = sys.modules['__main__'] # Unfortunately, the main ipython launch script historically had no # "if __name__ == '__main__'" guard, so we work around that # by treating it like a __main__.py file # See https://github.com/ipython/ipython/issues/4698 main_name = os.path.splitext(os.path.basename(main_path))[0] if main_name == 'ipython': return # Otherwise, if __file__ already has the setting we expect, # there's nothing more to do if getattr(current_main, '__file__', None) == main_path: return # If the parent process has sent a path through rather than a module # name we assume it is an executable script that may contain # non-main code that needs to be executed old_main_modules.append(current_main) main_module = types.ModuleType("__mp_main__") main_content = runpy.run_path(main_path, run_name="__mp_main__") main_module.__dict__.update(main_content) sys.modules['__main__'] = sys.modules['__mp_main__'] = main_module def import_main_path(main_path): ''' Set sys.modules['__main__'] to module at main_path ''' _fixup_main_from_path(main_path) managers.py 0000644 00000112407 15204243013 0006712 0 ustar 00 # # Module providing the `SyncManager` class for dealing # with shared objects # # multiprocessing/managers.py # # Copyright (c) 2006-2008, R Oudkerk # Licensed to PSF under a Contributor Agreement. # __all__ = [ 'BaseManager', 'SyncManager', 'BaseProxy', 'Token' ] # # Imports # import sys import threading import array import queue import time from traceback import format_exc from . import connection from .context import reduction, get_spawning_popen from . import pool from . import process from . import util from . import get_context # # Register some things for pickling # def reduce_array(a): return array.array, (a.typecode, a.tobytes()) reduction.register(array.array, reduce_array) view_types = [type(getattr({}, name)()) for name in ('items','keys','values')] if view_types[0] is not list: # only needed in Py3.0 def rebuild_as_list(obj): return list, (list(obj),) for view_type in view_types: reduction.register(view_type, rebuild_as_list) # # Type for identifying shared objects # class Token(object): ''' Type to uniquely indentify a shared object ''' __slots__ = ('typeid', 'address', 'id') def __init__(self, typeid, address, id): (self.typeid, self.address, self.id) = (typeid, address, id) def __getstate__(self): return (self.typeid, self.address, self.id) def __setstate__(self, state): (self.typeid, self.address, self.id) = state def __repr__(self): return '%s(typeid=%r, address=%r, id=%r)' % \ (self.__class__.__name__, self.typeid, self.address, self.id) # # Function for communication with a manager's server process # def dispatch(c, id, methodname, args=(), kwds={}): ''' Send a message to manager using connection `c` and return response ''' c.send((id, methodname, args, kwds)) kind, result = c.recv() if kind == '#RETURN': return result raise convert_to_error(kind, result) def convert_to_error(kind, result): if kind == '#ERROR': return result elif kind == '#TRACEBACK': assert type(result) is str return RemoteError(result) elif kind == '#UNSERIALIZABLE': assert type(result) is str return RemoteError('Unserializable message: %s\n' % result) else: return ValueError('Unrecognized message type') class RemoteError(Exception): def __str__(self): return ('\n' + '-'*75 + '\n' + str(self.args[0]) + '-'*75) # # Functions for finding the method names of an object # def all_methods(obj): ''' Return a list of names of methods of `obj` ''' temp = [] for name in dir(obj): func = getattr(obj, name) if callable(func): temp.append(name) return temp def public_methods(obj): ''' Return a list of names of methods of `obj` which do not start with '_' ''' return [name for name in all_methods(obj) if name[0] != '_'] # # Server which is run in a process controlled by a manager # class Server(object): ''' Server class which runs in a process controlled by a manager object ''' public = ['shutdown', 'create', 'accept_connection', 'get_methods', 'debug_info', 'number_of_objects', 'dummy', 'incref', 'decref'] def __init__(self, registry, address, authkey, serializer): assert isinstance(authkey, bytes) self.registry = registry self.authkey = process.AuthenticationString(authkey) Listener, Client = listener_client[serializer] # do authentication later self.listener = Listener(address=address, backlog=16) self.address = self.listener.address self.id_to_obj = {'0': (None, ())} self.id_to_refcount = {} self.id_to_local_proxy_obj = {} self.mutex = threading.Lock() def serve_forever(self): ''' Run the server forever ''' self.stop_event = threading.Event() process.current_process()._manager_server = self try: accepter = threading.Thread(target=self.accepter) accepter.daemon = True accepter.start() try: while not self.stop_event.is_set(): self.stop_event.wait(1) except (KeyboardInterrupt, SystemExit): pass finally: if sys.stdout != sys.__stdout__: util.debug('resetting stdout, stderr') sys.stdout = sys.__stdout__ sys.stderr = sys.__stderr__ sys.exit(0) def accepter(self): while True: try: c = self.listener.accept() except OSError: continue t = threading.Thread(target=self.handle_request, args=(c,)) t.daemon = True t.start() def handle_request(self, c): ''' Handle a new connection ''' funcname = result = request = None try: connection.deliver_challenge(c, self.authkey) connection.answer_challenge(c, self.authkey) request = c.recv() ignore, funcname, args, kwds = request assert funcname in self.public, '%r unrecognized' % funcname func = getattr(self, funcname) except Exception: msg = ('#TRACEBACK', format_exc()) else: try: result = func(c, *args, **kwds) except Exception: msg = ('#TRACEBACK', format_exc()) else: msg = ('#RETURN', result) try: c.send(msg) except Exception as e: try: c.send(('#TRACEBACK', format_exc())) except Exception: pass util.info('Failure to send message: %r', msg) util.info(' ... request was %r', request) util.info(' ... exception was %r', e) c.close() def serve_client(self, conn): ''' Handle requests from the proxies in a particular process/thread ''' util.debug('starting server thread to service %r', threading.current_thread().name) recv = conn.recv send = conn.send id_to_obj = self.id_to_obj while not self.stop_event.is_set(): try: methodname = obj = None request = recv() ident, methodname, args, kwds = request try: obj, exposed, gettypeid = id_to_obj[ident] except KeyError as ke: try: obj, exposed, gettypeid = \ self.id_to_local_proxy_obj[ident] except KeyError as second_ke: raise ke if methodname not in exposed: raise AttributeError( 'method %r of %r object is not in exposed=%r' % (methodname, type(obj), exposed) ) function = getattr(obj, methodname) try: res = function(*args, **kwds) except Exception as e: msg = ('#ERROR', e) else: typeid = gettypeid and gettypeid.get(methodname, None) if typeid: rident, rexposed = self.create(conn, typeid, res) token = Token(typeid, self.address, rident) msg = ('#PROXY', (rexposed, token)) else: msg = ('#RETURN', res) except AttributeError: if methodname is None: msg = ('#TRACEBACK', format_exc()) else: try: fallback_func = self.fallback_mapping[methodname] result = fallback_func( self, conn, ident, obj, *args, **kwds ) msg = ('#RETURN', result) except Exception: msg = ('#TRACEBACK', format_exc()) except EOFError: util.debug('got EOF -- exiting thread serving %r', threading.current_thread().name) sys.exit(0) except Exception: msg = ('#TRACEBACK', format_exc()) try: try: send(msg) except Exception as e: send(('#UNSERIALIZABLE', format_exc())) except Exception as e: util.info('exception in thread serving %r', threading.current_thread().name) util.info(' ... message was %r', msg) util.info(' ... exception was %r', e) conn.close() sys.exit(1) def fallback_getvalue(self, conn, ident, obj): return obj def fallback_str(self, conn, ident, obj): return str(obj) def fallback_repr(self, conn, ident, obj): return repr(obj) fallback_mapping = { '__str__':fallback_str, '__repr__':fallback_repr, '#GETVALUE':fallback_getvalue } def dummy(self, c): pass def debug_info(self, c): ''' Return some info --- useful to spot problems with refcounting ''' with self.mutex: result = [] keys = list(self.id_to_refcount.keys()) keys.sort() for ident in keys: if ident != '0': result.append(' %s: refcount=%s\n %s' % (ident, self.id_to_refcount[ident], str(self.id_to_obj[ident][0])[:75])) return '\n'.join(result) def number_of_objects(self, c): ''' Number of shared objects ''' # Doesn't use (len(self.id_to_obj) - 1) as we shouldn't count ident='0' return len(self.id_to_refcount) def shutdown(self, c): ''' Shutdown this process ''' try: util.debug('manager received shutdown message') c.send(('#RETURN', None)) except: import traceback traceback.print_exc() finally: self.stop_event.set() def create(self, c, typeid, *args, **kwds): ''' Create a new shared object and return its id ''' with self.mutex: callable, exposed, method_to_typeid, proxytype = \ self.registry[typeid] if callable is None: assert len(args) == 1 and not kwds obj = args[0] else: obj = callable(*args, **kwds) if exposed is None: exposed = public_methods(obj) if method_to_typeid is not None: assert type(method_to_typeid) is dict exposed = list(exposed) + list(method_to_typeid) ident = '%x' % id(obj) # convert to string because xmlrpclib # only has 32 bit signed integers util.debug('%r callable returned object with id %r', typeid, ident) self.id_to_obj[ident] = (obj, set(exposed), method_to_typeid) if ident not in self.id_to_refcount: self.id_to_refcount[ident] = 0 self.incref(c, ident) return ident, tuple(exposed) def get_methods(self, c, token): ''' Return the methods of the shared object indicated by token ''' return tuple(self.id_to_obj[token.id][1]) def accept_connection(self, c, name): ''' Spawn a new thread to serve this connection ''' threading.current_thread().name = name c.send(('#RETURN', None)) self.serve_client(c) def incref(self, c, ident): with self.mutex: try: self.id_to_refcount[ident] += 1 except KeyError as ke: # If no external references exist but an internal (to the # manager) still does and a new external reference is created # from it, restore the manager's tracking of it from the # previously stashed internal ref. if ident in self.id_to_local_proxy_obj: self.id_to_refcount[ident] = 1 self.id_to_obj[ident] = \ self.id_to_local_proxy_obj[ident] obj, exposed, gettypeid = self.id_to_obj[ident] util.debug('Server re-enabled tracking & INCREF %r', ident) else: raise ke def decref(self, c, ident): if ident not in self.id_to_refcount and \ ident in self.id_to_local_proxy_obj: util.debug('Server DECREF skipping %r', ident) return with self.mutex: assert self.id_to_refcount[ident] >= 1 self.id_to_refcount[ident] -= 1 if self.id_to_refcount[ident] == 0: del self.id_to_refcount[ident] if ident not in self.id_to_refcount: # Two-step process in case the object turns out to contain other # proxy objects (e.g. a managed list of managed lists). # Otherwise, deleting self.id_to_obj[ident] would trigger the # deleting of the stored value (another managed object) which would # in turn attempt to acquire the mutex that is already held here. self.id_to_obj[ident] = (None, (), None) # thread-safe util.debug('disposing of obj with id %r', ident) with self.mutex: del self.id_to_obj[ident] # # Class to represent state of a manager # class State(object): __slots__ = ['value'] INITIAL = 0 STARTED = 1 SHUTDOWN = 2 # # Mapping from serializer name to Listener and Client types # listener_client = { 'pickle' : (connection.Listener, connection.Client), 'xmlrpclib' : (connection.XmlListener, connection.XmlClient) } # # Definition of BaseManager # class BaseManager(object): ''' Base class for managers ''' _registry = {} _Server = Server def __init__(self, address=None, authkey=None, serializer='pickle', ctx=None): if authkey is None: authkey = process.current_process().authkey self._address = address # XXX not final address if eg ('', 0) self._authkey = process.AuthenticationString(authkey) self._state = State() self._state.value = State.INITIAL self._serializer = serializer self._Listener, self._Client = listener_client[serializer] self._ctx = ctx or get_context() def get_server(self): ''' Return server object with serve_forever() method and address attribute ''' assert self._state.value == State.INITIAL return Server(self._registry, self._address, self._authkey, self._serializer) def connect(self): ''' Connect manager object to the server process ''' Listener, Client = listener_client[self._serializer] conn = Client(self._address, authkey=self._authkey) dispatch(conn, None, 'dummy') self._state.value = State.STARTED def start(self, initializer=None, initargs=()): ''' Spawn a server process for this manager object ''' assert self._state.value == State.INITIAL if initializer is not None and not callable(initializer): raise TypeError('initializer must be a callable') # pipe over which we will retrieve address of server reader, writer = connection.Pipe(duplex=False) # spawn process which runs a server self._process = self._ctx.Process( target=type(self)._run_server, args=(self._registry, self._address, self._authkey, self._serializer, writer, initializer, initargs), ) ident = ':'.join(str(i) for i in self._process._identity) self._process.name = type(self).__name__ + '-' + ident self._process.start() # get address of server writer.close() self._address = reader.recv() reader.close() # register a finalizer self._state.value = State.STARTED self.shutdown = util.Finalize( self, type(self)._finalize_manager, args=(self._process, self._address, self._authkey, self._state, self._Client), exitpriority=0 ) @classmethod def _run_server(cls, registry, address, authkey, serializer, writer, initializer=None, initargs=()): ''' Create a server, report its address and run it ''' if initializer is not None: initializer(*initargs) # create server server = cls._Server(registry, address, authkey, serializer) # inform parent process of the server's address writer.send(server.address) writer.close() # run the manager util.info('manager serving at %r', server.address) server.serve_forever() def _create(self, typeid, *args, **kwds): ''' Create a new shared object; return the token and exposed tuple ''' assert self._state.value == State.STARTED, 'server not yet started' conn = self._Client(self._address, authkey=self._authkey) try: id, exposed = dispatch(conn, None, 'create', (typeid,)+args, kwds) finally: conn.close() return Token(typeid, self._address, id), exposed def join(self, timeout=None): ''' Join the manager process (if it has been spawned) ''' if self._process is not None: self._process.join(timeout) if not self._process.is_alive(): self._process = None def _debug_info(self): ''' Return some info about the servers shared objects and connections ''' conn = self._Client(self._address, authkey=self._authkey) try: return dispatch(conn, None, 'debug_info') finally: conn.close() def _number_of_objects(self): ''' Return the number of shared objects ''' conn = self._Client(self._address, authkey=self._authkey) try: return dispatch(conn, None, 'number_of_objects') finally: conn.close() def __enter__(self): if self._state.value == State.INITIAL: self.start() assert self._state.value == State.STARTED return self def __exit__(self, exc_type, exc_val, exc_tb): self.shutdown() @staticmethod def _finalize_manager(process, address, authkey, state, _Client): ''' Shutdown the manager process; will be registered as a finalizer ''' if process.is_alive(): util.info('sending shutdown message to manager') try: conn = _Client(address, authkey=authkey) try: dispatch(conn, None, 'shutdown') finally: conn.close() except Exception: pass process.join(timeout=1.0) if process.is_alive(): util.info('manager still alive') if hasattr(process, 'terminate'): util.info('trying to `terminate()` manager process') process.terminate() process.join(timeout=0.1) if process.is_alive(): util.info('manager still alive after terminate') state.value = State.SHUTDOWN try: del BaseProxy._address_to_local[address] except KeyError: pass address = property(lambda self: self._address) @classmethod def register(cls, typeid, callable=None, proxytype=None, exposed=None, method_to_typeid=None, create_method=True): ''' Register a typeid with the manager type ''' if '_registry' not in cls.__dict__: cls._registry = cls._registry.copy() if proxytype is None: proxytype = AutoProxy exposed = exposed or getattr(proxytype, '_exposed_', None) method_to_typeid = method_to_typeid or \ getattr(proxytype, '_method_to_typeid_', None) if method_to_typeid: for key, value in list(method_to_typeid.items()): assert type(key) is str, '%r is not a string' % key assert type(value) is str, '%r is not a string' % value cls._registry[typeid] = ( callable, exposed, method_to_typeid, proxytype ) if create_method: def temp(self, *args, **kwds): util.debug('requesting creation of a shared %r object', typeid) token, exp = self._create(typeid, *args, **kwds) proxy = proxytype( token, self._serializer, manager=self, authkey=self._authkey, exposed=exp ) conn = self._Client(token.address, authkey=self._authkey) dispatch(conn, None, 'decref', (token.id,)) return proxy temp.__name__ = typeid setattr(cls, typeid, temp) # # Subclass of set which get cleared after a fork # class ProcessLocalSet(set): def __init__(self): util.register_after_fork(self, lambda obj: obj.clear()) def __reduce__(self): return type(self), () # # Definition of BaseProxy # class BaseProxy(object): ''' A base for proxies of shared objects ''' _address_to_local = {} _mutex = util.ForkAwareThreadLock() def __init__(self, token, serializer, manager=None, authkey=None, exposed=None, incref=True, manager_owned=False): with BaseProxy._mutex: tls_idset = BaseProxy._address_to_local.get(token.address, None) if tls_idset is None: tls_idset = util.ForkAwareLocal(), ProcessLocalSet() BaseProxy._address_to_local[token.address] = tls_idset # self._tls is used to record the connection used by this # thread to communicate with the manager at token.address self._tls = tls_idset[0] # self._idset is used to record the identities of all shared # objects for which the current process owns references and # which are in the manager at token.address self._idset = tls_idset[1] self._token = token self._id = self._token.id self._manager = manager self._serializer = serializer self._Client = listener_client[serializer][1] # Should be set to True only when a proxy object is being created # on the manager server; primary use case: nested proxy objects. # RebuildProxy detects when a proxy is being created on the manager # and sets this value appropriately. self._owned_by_manager = manager_owned if authkey is not None: self._authkey = process.AuthenticationString(authkey) elif self._manager is not None: self._authkey = self._manager._authkey else: self._authkey = process.current_process().authkey if incref: self._incref() util.register_after_fork(self, BaseProxy._after_fork) def _connect(self): util.debug('making connection to manager') name = process.current_process().name if threading.current_thread().name != 'MainThread': name += '|' + threading.current_thread().name conn = self._Client(self._token.address, authkey=self._authkey) dispatch(conn, None, 'accept_connection', (name,)) self._tls.connection = conn def _callmethod(self, methodname, args=(), kwds={}): ''' Try to call a method of the referrent and return a copy of the result ''' try: conn = self._tls.connection except AttributeError: util.debug('thread %r does not own a connection', threading.current_thread().name) self._connect() conn = self._tls.connection conn.send((self._id, methodname, args, kwds)) kind, result = conn.recv() if kind == '#RETURN': return result elif kind == '#PROXY': exposed, token = result proxytype = self._manager._registry[token.typeid][-1] token.address = self._token.address proxy = proxytype( token, self._serializer, manager=self._manager, authkey=self._authkey, exposed=exposed ) conn = self._Client(token.address, authkey=self._authkey) dispatch(conn, None, 'decref', (token.id,)) return proxy raise convert_to_error(kind, result) def _getvalue(self): ''' Get a copy of the value of the referent ''' return self._callmethod('#GETVALUE') def _incref(self): if self._owned_by_manager: util.debug('owned_by_manager skipped INCREF of %r', self._token.id) return conn = self._Client(self._token.address, authkey=self._authkey) dispatch(conn, None, 'incref', (self._id,)) util.debug('INCREF %r', self._token.id) self._idset.add(self._id) state = self._manager and self._manager._state self._close = util.Finalize( self, BaseProxy._decref, args=(self._token, self._authkey, state, self._tls, self._idset, self._Client), exitpriority=10 ) @staticmethod def _decref(token, authkey, state, tls, idset, _Client): idset.discard(token.id) # check whether manager is still alive if state is None or state.value == State.STARTED: # tell manager this process no longer cares about referent try: util.debug('DECREF %r', token.id) conn = _Client(token.address, authkey=authkey) dispatch(conn, None, 'decref', (token.id,)) except Exception as e: util.debug('... decref failed %s', e) else: util.debug('DECREF %r -- manager already shutdown', token.id) # check whether we can close this thread's connection because # the process owns no more references to objects for this manager if not idset and hasattr(tls, 'connection'): util.debug('thread %r has no more proxies so closing conn', threading.current_thread().name) tls.connection.close() del tls.connection def _after_fork(self): self._manager = None try: self._incref() except Exception as e: # the proxy may just be for a manager which has shutdown util.info('incref failed: %s' % e) def __reduce__(self): kwds = {} if get_spawning_popen() is not None: kwds['authkey'] = self._authkey if getattr(self, '_isauto', False): kwds['exposed'] = self._exposed_ return (RebuildProxy, (AutoProxy, self._token, self._serializer, kwds)) else: return (RebuildProxy, (type(self), self._token, self._serializer, kwds)) def __deepcopy__(self, memo): return self._getvalue() def __repr__(self): return '<%s object, typeid %r at %#x>' % \ (type(self).__name__, self._token.typeid, id(self)) def __str__(self): ''' Return representation of the referent (or a fall-back if that fails) ''' try: return self._callmethod('__repr__') except Exception: return repr(self)[:-1] + "; '__str__()' failed>" # # Function used for unpickling # def RebuildProxy(func, token, serializer, kwds): ''' Function used for unpickling proxy objects. ''' server = getattr(process.current_process(), '_manager_server', None) if server and server.address == token.address: util.debug('Rebuild a proxy owned by manager, token=%r', token) kwds['manager_owned'] = True if token.id not in server.id_to_local_proxy_obj: server.id_to_local_proxy_obj[token.id] = \ server.id_to_obj[token.id] incref = ( kwds.pop('incref', True) and not getattr(process.current_process(), '_inheriting', False) ) return func(token, serializer, incref=incref, **kwds) # # Functions to create proxies and proxy types # def MakeProxyType(name, exposed, _cache={}): ''' Return a proxy type whose methods are given by `exposed` ''' exposed = tuple(exposed) try: return _cache[(name, exposed)] except KeyError: pass dic = {} for meth in exposed: exec('''def %s(self, *args, **kwds): return self._callmethod(%r, args, kwds)''' % (meth, meth), dic) ProxyType = type(name, (BaseProxy,), dic) ProxyType._exposed_ = exposed _cache[(name, exposed)] = ProxyType return ProxyType def AutoProxy(token, serializer, manager=None, authkey=None, exposed=None, incref=True): ''' Return an auto-proxy for `token` ''' _Client = listener_client[serializer][1] if exposed is None: conn = _Client(token.address, authkey=authkey) try: exposed = dispatch(conn, None, 'get_methods', (token,)) finally: conn.close() if authkey is None and manager is not None: authkey = manager._authkey if authkey is None: authkey = process.current_process().authkey ProxyType = MakeProxyType('AutoProxy[%s]' % token.typeid, exposed) proxy = ProxyType(token, serializer, manager=manager, authkey=authkey, incref=incref) proxy._isauto = True return proxy # # Types/callables which we will register with SyncManager # class Namespace(object): def __init__(self, **kwds): self.__dict__.update(kwds) def __repr__(self): items = list(self.__dict__.items()) temp = [] for name, value in items: if not name.startswith('_'): temp.append('%s=%r' % (name, value)) temp.sort() return '%s(%s)' % (self.__class__.__name__, ', '.join(temp)) class Value(object): def __init__(self, typecode, value, lock=True): self._typecode = typecode self._value = value def get(self): return self._value def set(self, value): self._value = value def __repr__(self): return '%s(%r, %r)'%(type(self).__name__, self._typecode, self._value) value = property(get, set) def Array(typecode, sequence, lock=True): return array.array(typecode, sequence) # # Proxy types used by SyncManager # class IteratorProxy(BaseProxy): _exposed_ = ('__next__', 'send', 'throw', 'close') def __iter__(self): return self def __next__(self, *args): return self._callmethod('__next__', args) def send(self, *args): return self._callmethod('send', args) def throw(self, *args): return self._callmethod('throw', args) def close(self, *args): return self._callmethod('close', args) class AcquirerProxy(BaseProxy): _exposed_ = ('acquire', 'release') def acquire(self, blocking=True, timeout=None): args = (blocking,) if timeout is None else (blocking, timeout) return self._callmethod('acquire', args) def release(self): return self._callmethod('release') def __enter__(self): return self._callmethod('acquire') def __exit__(self, exc_type, exc_val, exc_tb): return self._callmethod('release') class ConditionProxy(AcquirerProxy): _exposed_ = ('acquire', 'release', 'wait', 'notify', 'notify_all') def wait(self, timeout=None): return self._callmethod('wait', (timeout,)) def notify(self): return self._callmethod('notify') def notify_all(self): return self._callmethod('notify_all') def wait_for(self, predicate, timeout=None): result = predicate() if result: return result if timeout is not None: endtime = time.monotonic() + timeout else: endtime = None waittime = None while not result: if endtime is not None: waittime = endtime - time.monotonic() if waittime <= 0: break self.wait(waittime) result = predicate() return result class EventProxy(BaseProxy): _exposed_ = ('is_set', 'set', 'clear', 'wait') def is_set(self): return self._callmethod('is_set') def set(self): return self._callmethod('set') def clear(self): return self._callmethod('clear') def wait(self, timeout=None): return self._callmethod('wait', (timeout,)) class BarrierProxy(BaseProxy): _exposed_ = ('__getattribute__', 'wait', 'abort', 'reset') def wait(self, timeout=None): return self._callmethod('wait', (timeout,)) def abort(self): return self._callmethod('abort') def reset(self): return self._callmethod('reset') @property def parties(self): return self._callmethod('__getattribute__', ('parties',)) @property def n_waiting(self): return self._callmethod('__getattribute__', ('n_waiting',)) @property def broken(self): return self._callmethod('__getattribute__', ('broken',)) class NamespaceProxy(BaseProxy): _exposed_ = ('__getattribute__', '__setattr__', '__delattr__') def __getattr__(self, key): if key[0] == '_': return object.__getattribute__(self, key) callmethod = object.__getattribute__(self, '_callmethod') return callmethod('__getattribute__', (key,)) def __setattr__(self, key, value): if key[0] == '_': return object.__setattr__(self, key, value) callmethod = object.__getattribute__(self, '_callmethod') return callmethod('__setattr__', (key, value)) def __delattr__(self, key): if key[0] == '_': return object.__delattr__(self, key) callmethod = object.__getattribute__(self, '_callmethod') return callmethod('__delattr__', (key,)) class ValueProxy(BaseProxy): _exposed_ = ('get', 'set') def get(self): return self._callmethod('get') def set(self, value): return self._callmethod('set', (value,)) value = property(get, set) BaseListProxy = MakeProxyType('BaseListProxy', ( '__add__', '__contains__', '__delitem__', '__getitem__', '__len__', '__mul__', '__reversed__', '__rmul__', '__setitem__', 'append', 'count', 'extend', 'index', 'insert', 'pop', 'remove', 'reverse', 'sort', '__imul__' )) class ListProxy(BaseListProxy): def __iadd__(self, value): self._callmethod('extend', (value,)) return self def __imul__(self, value): self._callmethod('__imul__', (value,)) return self DictProxy = MakeProxyType('DictProxy', ( '__contains__', '__delitem__', '__getitem__', '__iter__', '__len__', '__setitem__', 'clear', 'copy', 'get', 'has_key', 'items', 'keys', 'pop', 'popitem', 'setdefault', 'update', 'values' )) DictProxy._method_to_typeid_ = { '__iter__': 'Iterator', } ArrayProxy = MakeProxyType('ArrayProxy', ( '__len__', '__getitem__', '__setitem__' )) BasePoolProxy = MakeProxyType('PoolProxy', ( 'apply', 'apply_async', 'close', 'imap', 'imap_unordered', 'join', 'map', 'map_async', 'starmap', 'starmap_async', 'terminate', )) BasePoolProxy._method_to_typeid_ = { 'apply_async': 'AsyncResult', 'map_async': 'AsyncResult', 'starmap_async': 'AsyncResult', 'imap': 'Iterator', 'imap_unordered': 'Iterator' } class PoolProxy(BasePoolProxy): def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.terminate() # # Definition of SyncManager # class SyncManager(BaseManager): ''' Subclass of `BaseManager` which supports a number of shared object types. The types registered are those intended for the synchronization of threads, plus `dict`, `list` and `Namespace`. The `multiprocessing.Manager()` function creates started instances of this class. ''' SyncManager.register('Queue', queue.Queue) SyncManager.register('JoinableQueue', queue.Queue) SyncManager.register('Event', threading.Event, EventProxy) SyncManager.register('Lock', threading.Lock, AcquirerProxy) SyncManager.register('RLock', threading.RLock, AcquirerProxy) SyncManager.register('Semaphore', threading.Semaphore, AcquirerProxy) SyncManager.register('BoundedSemaphore', threading.BoundedSemaphore, AcquirerProxy) SyncManager.register('Condition', threading.Condition, ConditionProxy) SyncManager.register('Barrier', threading.Barrier, BarrierProxy) SyncManager.register('Pool', pool.Pool, PoolProxy) SyncManager.register('list', list, ListProxy) SyncManager.register('dict', dict, DictProxy) SyncManager.register('Value', Value, ValueProxy) SyncManager.register('Array', Array, ArrayProxy) SyncManager.register('Namespace', Namespace, NamespaceProxy) # types returned by methods of PoolProxy SyncManager.register('Iterator', proxytype=IteratorProxy, create_method=False) SyncManager.register('AsyncResult', create_method=False)
| ver. 1.4 |
Github
|
.
| PHP 8.1.34 | ���֧ߧ֧�ѧ�ڧ� ����ѧߧڧ��: 0.03 |
proxy
|
phpinfo
|
���ѧ����ۧܧ�