pkg_resources.py 99 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843
  1. """
  2. Package resource API
  3. --------------------
  4. A resource is a logical file contained within a package, or a logical
  5. subdirectory thereof. The package resource API expects resource names
  6. to have their path parts separated with ``/``, *not* whatever the local
  7. path separator is. Do not use os.path operations to manipulate resource
  8. names being passed into the API.
  9. The package resource API is designed to work with normal filesystem packages,
  10. .egg files, and unpacked .egg files. It can also work in a limited way with
  11. .zip files and with custom PEP 302 loaders that support the ``get_data()``
  12. method.
  13. """
  14. import sys
  15. import os
  16. import time
  17. import re
  18. import imp
  19. import zipfile
  20. import zipimport
  21. import warnings
  22. import stat
  23. import functools
  24. import pkgutil
  25. import token
  26. import symbol
  27. import operator
  28. import platform
  29. from pkgutil import get_importer
  30. try:
  31. from urlparse import urlparse, urlunparse
  32. except ImportError:
  33. from urllib.parse import urlparse, urlunparse
  34. try:
  35. frozenset
  36. except NameError:
  37. from sets import ImmutableSet as frozenset
  38. try:
  39. basestring
  40. next = lambda o: o.next()
  41. from cStringIO import StringIO as BytesIO
  42. except NameError:
  43. basestring = str
  44. from io import BytesIO
  45. def execfile(fn, globs=None, locs=None):
  46. if globs is None:
  47. globs = globals()
  48. if locs is None:
  49. locs = globs
  50. exec(compile(open(fn).read(), fn, 'exec'), globs, locs)
  51. # capture these to bypass sandboxing
  52. from os import utime
  53. try:
  54. from os import mkdir, rename, unlink
  55. WRITE_SUPPORT = True
  56. except ImportError:
  57. # no write support, probably under GAE
  58. WRITE_SUPPORT = False
  59. from os import open as os_open
  60. from os.path import isdir, split
  61. # Avoid try/except due to potential problems with delayed import mechanisms.
  62. if sys.version_info >= (3, 3) and sys.implementation.name == "cpython":
  63. import importlib._bootstrap as importlib_bootstrap
  64. else:
  65. importlib_bootstrap = None
  66. try:
  67. import parser
  68. except ImportError:
  69. pass
  70. def _bypass_ensure_directory(name, mode=0o777):
  71. # Sandbox-bypassing version of ensure_directory()
  72. if not WRITE_SUPPORT:
  73. raise IOError('"os.mkdir" not supported on this platform.')
  74. dirname, filename = split(name)
  75. if dirname and filename and not isdir(dirname):
  76. _bypass_ensure_directory(dirname)
  77. mkdir(dirname, mode)
  78. _state_vars = {}
  79. def _declare_state(vartype, **kw):
  80. globals().update(kw)
  81. _state_vars.update(dict.fromkeys(kw, vartype))
  82. def __getstate__():
  83. state = {}
  84. g = globals()
  85. for k, v in _state_vars.items():
  86. state[k] = g['_sget_'+v](g[k])
  87. return state
  88. def __setstate__(state):
  89. g = globals()
  90. for k, v in state.items():
  91. g['_sset_'+_state_vars[k]](k, g[k], v)
  92. return state
  93. def _sget_dict(val):
  94. return val.copy()
  95. def _sset_dict(key, ob, state):
  96. ob.clear()
  97. ob.update(state)
  98. def _sget_object(val):
  99. return val.__getstate__()
  100. def _sset_object(key, ob, state):
  101. ob.__setstate__(state)
  102. _sget_none = _sset_none = lambda *args: None
  103. def get_supported_platform():
  104. """Return this platform's maximum compatible version.
  105. distutils.util.get_platform() normally reports the minimum version
  106. of Mac OS X that would be required to *use* extensions produced by
  107. distutils. But what we want when checking compatibility is to know the
  108. version of Mac OS X that we are *running*. To allow usage of packages that
  109. explicitly require a newer version of Mac OS X, we must also know the
  110. current version of the OS.
  111. If this condition occurs for any other platform with a version in its
  112. platform strings, this function should be extended accordingly.
  113. """
  114. plat = get_build_platform()
  115. m = macosVersionString.match(plat)
  116. if m is not None and sys.platform == "darwin":
  117. try:
  118. plat = 'macosx-%s-%s' % ('.'.join(_macosx_vers()[:2]), m.group(3))
  119. except ValueError:
  120. # not Mac OS X
  121. pass
  122. return plat
  123. __all__ = [
  124. # Basic resource access and distribution/entry point discovery
  125. 'require', 'run_script', 'get_provider', 'get_distribution',
  126. 'load_entry_point', 'get_entry_map', 'get_entry_info',
  127. 'iter_entry_points',
  128. 'resource_string', 'resource_stream', 'resource_filename',
  129. 'resource_listdir', 'resource_exists', 'resource_isdir',
  130. # Environmental control
  131. 'declare_namespace', 'working_set', 'add_activation_listener',
  132. 'find_distributions', 'set_extraction_path', 'cleanup_resources',
  133. 'get_default_cache',
  134. # Primary implementation classes
  135. 'Environment', 'WorkingSet', 'ResourceManager',
  136. 'Distribution', 'Requirement', 'EntryPoint',
  137. # Exceptions
  138. 'ResolutionError', 'VersionConflict', 'DistributionNotFound',
  139. 'UnknownExtra', 'ExtractionError',
  140. # Parsing functions and string utilities
  141. 'parse_requirements', 'parse_version', 'safe_name', 'safe_version',
  142. 'get_platform', 'compatible_platforms', 'yield_lines', 'split_sections',
  143. 'safe_extra', 'to_filename', 'invalid_marker', 'evaluate_marker',
  144. # filesystem utilities
  145. 'ensure_directory', 'normalize_path',
  146. # Distribution "precedence" constants
  147. 'EGG_DIST', 'BINARY_DIST', 'SOURCE_DIST', 'CHECKOUT_DIST', 'DEVELOP_DIST',
  148. # "Provider" interfaces, implementations, and registration/lookup APIs
  149. 'IMetadataProvider', 'IResourceProvider', 'FileMetadata',
  150. 'PathMetadata', 'EggMetadata', 'EmptyProvider', 'empty_provider',
  151. 'NullProvider', 'EggProvider', 'DefaultProvider', 'ZipProvider',
  152. 'register_finder', 'register_namespace_handler', 'register_loader_type',
  153. 'fixup_namespace_packages', 'get_importer',
  154. # Deprecated/backward compatibility only
  155. 'run_main', 'AvailableDistributions',
  156. ]
  157. class ResolutionError(Exception):
  158. """Abstract base for dependency resolution errors"""
  159. def __repr__(self):
  160. return self.__class__.__name__+repr(self.args)
  161. class VersionConflict(ResolutionError):
  162. """An already-installed version conflicts with the requested version"""
  163. class DistributionNotFound(ResolutionError):
  164. """A requested distribution was not found"""
  165. class UnknownExtra(ResolutionError):
  166. """Distribution doesn't have an "extra feature" of the given name"""
  167. _provider_factories = {}
  168. PY_MAJOR = sys.version[:3]
  169. EGG_DIST = 3
  170. BINARY_DIST = 2
  171. SOURCE_DIST = 1
  172. CHECKOUT_DIST = 0
  173. DEVELOP_DIST = -1
  174. def register_loader_type(loader_type, provider_factory):
  175. """Register `provider_factory` to make providers for `loader_type`
  176. `loader_type` is the type or class of a PEP 302 ``module.__loader__``,
  177. and `provider_factory` is a function that, passed a *module* object,
  178. returns an ``IResourceProvider`` for that module.
  179. """
  180. _provider_factories[loader_type] = provider_factory
  181. def get_provider(moduleOrReq):
  182. """Return an IResourceProvider for the named module or requirement"""
  183. if isinstance(moduleOrReq, Requirement):
  184. return working_set.find(moduleOrReq) or require(str(moduleOrReq))[0]
  185. try:
  186. module = sys.modules[moduleOrReq]
  187. except KeyError:
  188. __import__(moduleOrReq)
  189. module = sys.modules[moduleOrReq]
  190. loader = getattr(module, '__loader__', None)
  191. return _find_adapter(_provider_factories, loader)(module)
  192. def _macosx_vers(_cache=[]):
  193. if not _cache:
  194. import platform
  195. version = platform.mac_ver()[0]
  196. # fallback for MacPorts
  197. if version == '':
  198. import plistlib
  199. plist = '/System/Library/CoreServices/SystemVersion.plist'
  200. if os.path.exists(plist):
  201. if hasattr(plistlib, 'readPlist'):
  202. plist_content = plistlib.readPlist(plist)
  203. if 'ProductVersion' in plist_content:
  204. version = plist_content['ProductVersion']
  205. _cache.append(version.split('.'))
  206. return _cache[0]
  207. def _macosx_arch(machine):
  208. return {'PowerPC': 'ppc', 'Power_Macintosh': 'ppc'}.get(machine, machine)
  209. def get_build_platform():
  210. """Return this platform's string for platform-specific distributions
  211. XXX Currently this is the same as ``distutils.util.get_platform()``, but it
  212. needs some hacks for Linux and Mac OS X.
  213. """
  214. try:
  215. # Python 2.7 or >=3.2
  216. from sysconfig import get_platform
  217. except ImportError:
  218. from distutils.util import get_platform
  219. plat = get_platform()
  220. if sys.platform == "darwin" and not plat.startswith('macosx-'):
  221. try:
  222. version = _macosx_vers()
  223. machine = os.uname()[4].replace(" ", "_")
  224. return "macosx-%d.%d-%s" % (int(version[0]), int(version[1]),
  225. _macosx_arch(machine))
  226. except ValueError:
  227. # if someone is running a non-Mac darwin system, this will fall
  228. # through to the default implementation
  229. pass
  230. return plat
  231. macosVersionString = re.compile(r"macosx-(\d+)\.(\d+)-(.*)")
  232. darwinVersionString = re.compile(r"darwin-(\d+)\.(\d+)\.(\d+)-(.*)")
  233. # XXX backward compat
  234. get_platform = get_build_platform
  235. def compatible_platforms(provided, required):
  236. """Can code for the `provided` platform run on the `required` platform?
  237. Returns true if either platform is ``None``, or the platforms are equal.
  238. XXX Needs compatibility checks for Linux and other unixy OSes.
  239. """
  240. if provided is None or required is None or provided==required:
  241. # easy case
  242. return True
  243. # Mac OS X special cases
  244. reqMac = macosVersionString.match(required)
  245. if reqMac:
  246. provMac = macosVersionString.match(provided)
  247. # is this a Mac package?
  248. if not provMac:
  249. # this is backwards compatibility for packages built before
  250. # setuptools 0.6. All packages built after this point will
  251. # use the new macosx designation.
  252. provDarwin = darwinVersionString.match(provided)
  253. if provDarwin:
  254. dversion = int(provDarwin.group(1))
  255. macosversion = "%s.%s" % (reqMac.group(1), reqMac.group(2))
  256. if dversion == 7 and macosversion >= "10.3" or \
  257. dversion == 8 and macosversion >= "10.4":
  258. #import warnings
  259. #warnings.warn("Mac eggs should be rebuilt to "
  260. # "use the macosx designation instead of darwin.",
  261. # category=DeprecationWarning)
  262. return True
  263. return False # egg isn't macosx or legacy darwin
  264. # are they the same major version and machine type?
  265. if provMac.group(1) != reqMac.group(1) or \
  266. provMac.group(3) != reqMac.group(3):
  267. return False
  268. # is the required OS major update >= the provided one?
  269. if int(provMac.group(2)) > int(reqMac.group(2)):
  270. return False
  271. return True
  272. # XXX Linux and other platforms' special cases should go here
  273. return False
  274. def run_script(dist_spec, script_name):
  275. """Locate distribution `dist_spec` and run its `script_name` script"""
  276. ns = sys._getframe(1).f_globals
  277. name = ns['__name__']
  278. ns.clear()
  279. ns['__name__'] = name
  280. require(dist_spec)[0].run_script(script_name, ns)
  281. # backward compatibility
  282. run_main = run_script
  283. def get_distribution(dist):
  284. """Return a current distribution object for a Requirement or string"""
  285. if isinstance(dist, basestring): dist = Requirement.parse(dist)
  286. if isinstance(dist, Requirement): dist = get_provider(dist)
  287. if not isinstance(dist, Distribution):
  288. raise TypeError("Expected string, Requirement, or Distribution", dist)
  289. return dist
  290. def load_entry_point(dist, group, name):
  291. """Return `name` entry point of `group` for `dist` or raise ImportError"""
  292. return get_distribution(dist).load_entry_point(group, name)
  293. def get_entry_map(dist, group=None):
  294. """Return the entry point map for `group`, or the full entry map"""
  295. return get_distribution(dist).get_entry_map(group)
  296. def get_entry_info(dist, group, name):
  297. """Return the EntryPoint object for `group`+`name`, or ``None``"""
  298. return get_distribution(dist).get_entry_info(group, name)
  299. class IMetadataProvider:
  300. def has_metadata(name):
  301. """Does the package's distribution contain the named metadata?"""
  302. def get_metadata(name):
  303. """The named metadata resource as a string"""
  304. def get_metadata_lines(name):
  305. """Yield named metadata resource as list of non-blank non-comment lines
  306. Leading and trailing whitespace is stripped from each line, and lines
  307. with ``#`` as the first non-blank character are omitted."""
  308. def metadata_isdir(name):
  309. """Is the named metadata a directory? (like ``os.path.isdir()``)"""
  310. def metadata_listdir(name):
  311. """List of metadata names in the directory (like ``os.listdir()``)"""
  312. def run_script(script_name, namespace):
  313. """Execute the named script in the supplied namespace dictionary"""
  314. class IResourceProvider(IMetadataProvider):
  315. """An object that provides access to package resources"""
  316. def get_resource_filename(manager, resource_name):
  317. """Return a true filesystem path for `resource_name`
  318. `manager` must be an ``IResourceManager``"""
  319. def get_resource_stream(manager, resource_name):
  320. """Return a readable file-like object for `resource_name`
  321. `manager` must be an ``IResourceManager``"""
  322. def get_resource_string(manager, resource_name):
  323. """Return a string containing the contents of `resource_name`
  324. `manager` must be an ``IResourceManager``"""
  325. def has_resource(resource_name):
  326. """Does the package contain the named resource?"""
  327. def resource_isdir(resource_name):
  328. """Is the named resource a directory? (like ``os.path.isdir()``)"""
  329. def resource_listdir(resource_name):
  330. """List of resource names in the directory (like ``os.listdir()``)"""
  331. class WorkingSet(object):
  332. """A collection of active distributions on sys.path (or a similar list)"""
  333. def __init__(self, entries=None):
  334. """Create working set from list of path entries (default=sys.path)"""
  335. self.entries = []
  336. self.entry_keys = {}
  337. self.by_key = {}
  338. self.callbacks = []
  339. if entries is None:
  340. entries = sys.path
  341. for entry in entries:
  342. self.add_entry(entry)
  343. @classmethod
  344. def _build_master(cls):
  345. """
  346. Prepare the master working set.
  347. """
  348. ws = cls()
  349. try:
  350. from __main__ import __requires__
  351. except ImportError:
  352. # The main program does not list any requirements
  353. return ws
  354. # ensure the requirements are met
  355. try:
  356. ws.require(__requires__)
  357. except VersionConflict:
  358. return cls._build_from_requirements(__requires__)
  359. return ws
  360. @classmethod
  361. def _build_from_requirements(cls, req_spec):
  362. """
  363. Build a working set from a requirement spec. Rewrites sys.path.
  364. """
  365. # try it without defaults already on sys.path
  366. # by starting with an empty path
  367. ws = cls([])
  368. reqs = parse_requirements(req_spec)
  369. dists = ws.resolve(reqs, Environment())
  370. for dist in dists:
  371. ws.add(dist)
  372. # add any missing entries from sys.path
  373. for entry in sys.path:
  374. if entry not in ws.entries:
  375. ws.add_entry(entry)
  376. # then copy back to sys.path
  377. sys.path[:] = ws.entries
  378. return ws
  379. def add_entry(self, entry):
  380. """Add a path item to ``.entries``, finding any distributions on it
  381. ``find_distributions(entry, True)`` is used to find distributions
  382. corresponding to the path entry, and they are added. `entry` is
  383. always appended to ``.entries``, even if it is already present.
  384. (This is because ``sys.path`` can contain the same value more than
  385. once, and the ``.entries`` of the ``sys.path`` WorkingSet should always
  386. equal ``sys.path``.)
  387. """
  388. self.entry_keys.setdefault(entry, [])
  389. self.entries.append(entry)
  390. for dist in find_distributions(entry, True):
  391. self.add(dist, entry, False)
  392. def __contains__(self, dist):
  393. """True if `dist` is the active distribution for its project"""
  394. return self.by_key.get(dist.key) == dist
  395. def find(self, req):
  396. """Find a distribution matching requirement `req`
  397. If there is an active distribution for the requested project, this
  398. returns it as long as it meets the version requirement specified by
  399. `req`. But, if there is an active distribution for the project and it
  400. does *not* meet the `req` requirement, ``VersionConflict`` is raised.
  401. If there is no active distribution for the requested project, ``None``
  402. is returned.
  403. """
  404. dist = self.by_key.get(req.key)
  405. if dist is not None and dist not in req:
  406. # XXX add more info
  407. raise VersionConflict(dist, req)
  408. else:
  409. return dist
  410. def iter_entry_points(self, group, name=None):
  411. """Yield entry point objects from `group` matching `name`
  412. If `name` is None, yields all entry points in `group` from all
  413. distributions in the working set, otherwise only ones matching
  414. both `group` and `name` are yielded (in distribution order).
  415. """
  416. for dist in self:
  417. entries = dist.get_entry_map(group)
  418. if name is None:
  419. for ep in entries.values():
  420. yield ep
  421. elif name in entries:
  422. yield entries[name]
  423. def run_script(self, requires, script_name):
  424. """Locate distribution for `requires` and run `script_name` script"""
  425. ns = sys._getframe(1).f_globals
  426. name = ns['__name__']
  427. ns.clear()
  428. ns['__name__'] = name
  429. self.require(requires)[0].run_script(script_name, ns)
  430. def __iter__(self):
  431. """Yield distributions for non-duplicate projects in the working set
  432. The yield order is the order in which the items' path entries were
  433. added to the working set.
  434. """
  435. seen = {}
  436. for item in self.entries:
  437. if item not in self.entry_keys:
  438. # workaround a cache issue
  439. continue
  440. for key in self.entry_keys[item]:
  441. if key not in seen:
  442. seen[key]=1
  443. yield self.by_key[key]
  444. def add(self, dist, entry=None, insert=True, replace=False):
  445. """Add `dist` to working set, associated with `entry`
  446. If `entry` is unspecified, it defaults to the ``.location`` of `dist`.
  447. On exit from this routine, `entry` is added to the end of the working
  448. set's ``.entries`` (if it wasn't already present).
  449. `dist` is only added to the working set if it's for a project that
  450. doesn't already have a distribution in the set, unless `replace=True`.
  451. If it's added, any callbacks registered with the ``subscribe()`` method
  452. will be called.
  453. """
  454. if insert:
  455. dist.insert_on(self.entries, entry)
  456. if entry is None:
  457. entry = dist.location
  458. keys = self.entry_keys.setdefault(entry,[])
  459. keys2 = self.entry_keys.setdefault(dist.location,[])
  460. if not replace and dist.key in self.by_key:
  461. # ignore hidden distros
  462. return
  463. self.by_key[dist.key] = dist
  464. if dist.key not in keys:
  465. keys.append(dist.key)
  466. if dist.key not in keys2:
  467. keys2.append(dist.key)
  468. self._added_new(dist)
  469. def resolve(self, requirements, env=None, installer=None,
  470. replace_conflicting=False):
  471. """List all distributions needed to (recursively) meet `requirements`
  472. `requirements` must be a sequence of ``Requirement`` objects. `env`,
  473. if supplied, should be an ``Environment`` instance. If
  474. not supplied, it defaults to all distributions available within any
  475. entry or distribution in the working set. `installer`, if supplied,
  476. will be invoked with each requirement that cannot be met by an
  477. already-installed distribution; it should return a ``Distribution`` or
  478. ``None``.
  479. Unless `replace_conflicting=True`, raises a VersionConflict exception if
  480. any requirements are found on the path that have the correct name but
  481. the wrong version. Otherwise, if an `installer` is supplied it will be
  482. invoked to obtain the correct version of the requirement and activate
  483. it.
  484. """
  485. # set up the stack
  486. requirements = list(requirements)[::-1]
  487. # set of processed requirements
  488. processed = {}
  489. # key -> dist
  490. best = {}
  491. to_activate = []
  492. while requirements:
  493. # process dependencies breadth-first
  494. req = requirements.pop(0)
  495. if req in processed:
  496. # Ignore cyclic or redundant dependencies
  497. continue
  498. dist = best.get(req.key)
  499. if dist is None:
  500. # Find the best distribution and add it to the map
  501. dist = self.by_key.get(req.key)
  502. if dist is None or (dist not in req and replace_conflicting):
  503. ws = self
  504. if env is None:
  505. if dist is None:
  506. env = Environment(self.entries)
  507. else:
  508. # Use an empty environment and workingset to avoid
  509. # any further conflicts with the conflicting
  510. # distribution
  511. env = Environment([])
  512. ws = WorkingSet([])
  513. dist = best[req.key] = env.best_match(req, ws, installer)
  514. if dist is None:
  515. #msg = ("The '%s' distribution was not found on this "
  516. # "system, and is required by this application.")
  517. #raise DistributionNotFound(msg % req)
  518. # unfortunately, zc.buildout uses a str(err)
  519. # to get the name of the distribution here..
  520. raise DistributionNotFound(req)
  521. to_activate.append(dist)
  522. if dist not in req:
  523. # Oops, the "best" so far conflicts with a dependency
  524. raise VersionConflict(dist, req) # XXX put more info here
  525. requirements.extend(dist.requires(req.extras)[::-1])
  526. processed[req] = True
  527. # return list of distros to activate
  528. return to_activate
  529. def find_plugins(self, plugin_env, full_env=None, installer=None,
  530. fallback=True):
  531. """Find all activatable distributions in `plugin_env`
  532. Example usage::
  533. distributions, errors = working_set.find_plugins(
  534. Environment(plugin_dirlist)
  535. )
  536. map(working_set.add, distributions) # add plugins+libs to sys.path
  537. print 'Could not load', errors # display errors
  538. The `plugin_env` should be an ``Environment`` instance that contains
  539. only distributions that are in the project's "plugin directory" or
  540. directories. The `full_env`, if supplied, should be an ``Environment``
  541. contains all currently-available distributions. If `full_env` is not
  542. supplied, one is created automatically from the ``WorkingSet`` this
  543. method is called on, which will typically mean that every directory on
  544. ``sys.path`` will be scanned for distributions.
  545. `installer` is a standard installer callback as used by the
  546. ``resolve()`` method. The `fallback` flag indicates whether we should
  547. attempt to resolve older versions of a plugin if the newest version
  548. cannot be resolved.
  549. This method returns a 2-tuple: (`distributions`, `error_info`), where
  550. `distributions` is a list of the distributions found in `plugin_env`
  551. that were loadable, along with any other distributions that are needed
  552. to resolve their dependencies. `error_info` is a dictionary mapping
  553. unloadable plugin distributions to an exception instance describing the
  554. error that occurred. Usually this will be a ``DistributionNotFound`` or
  555. ``VersionConflict`` instance.
  556. """
  557. plugin_projects = list(plugin_env)
  558. # scan project names in alphabetic order
  559. plugin_projects.sort()
  560. error_info = {}
  561. distributions = {}
  562. if full_env is None:
  563. env = Environment(self.entries)
  564. env += plugin_env
  565. else:
  566. env = full_env + plugin_env
  567. shadow_set = self.__class__([])
  568. # put all our entries in shadow_set
  569. list(map(shadow_set.add, self))
  570. for project_name in plugin_projects:
  571. for dist in plugin_env[project_name]:
  572. req = [dist.as_requirement()]
  573. try:
  574. resolvees = shadow_set.resolve(req, env, installer)
  575. except ResolutionError:
  576. v = sys.exc_info()[1]
  577. # save error info
  578. error_info[dist] = v
  579. if fallback:
  580. # try the next older version of project
  581. continue
  582. else:
  583. # give up on this project, keep going
  584. break
  585. else:
  586. list(map(shadow_set.add, resolvees))
  587. distributions.update(dict.fromkeys(resolvees))
  588. # success, no need to try any more versions of this project
  589. break
  590. distributions = list(distributions)
  591. distributions.sort()
  592. return distributions, error_info
  593. def require(self, *requirements):
  594. """Ensure that distributions matching `requirements` are activated
  595. `requirements` must be a string or a (possibly-nested) sequence
  596. thereof, specifying the distributions and versions required. The
  597. return value is a sequence of the distributions that needed to be
  598. activated to fulfill the requirements; all relevant distributions are
  599. included, even if they were already activated in this working set.
  600. """
  601. needed = self.resolve(parse_requirements(requirements))
  602. for dist in needed:
  603. self.add(dist)
  604. return needed
  605. def subscribe(self, callback):
  606. """Invoke `callback` for all distributions (including existing ones)"""
  607. if callback in self.callbacks:
  608. return
  609. self.callbacks.append(callback)
  610. for dist in self:
  611. callback(dist)
  612. def _added_new(self, dist):
  613. for callback in self.callbacks:
  614. callback(dist)
  615. def __getstate__(self):
  616. return (
  617. self.entries[:], self.entry_keys.copy(), self.by_key.copy(),
  618. self.callbacks[:]
  619. )
  620. def __setstate__(self, e_k_b_c):
  621. entries, keys, by_key, callbacks = e_k_b_c
  622. self.entries = entries[:]
  623. self.entry_keys = keys.copy()
  624. self.by_key = by_key.copy()
  625. self.callbacks = callbacks[:]
  626. class Environment(object):
  627. """Searchable snapshot of distributions on a search path"""
  628. def __init__(self, search_path=None, platform=get_supported_platform(),
  629. python=PY_MAJOR):
  630. """Snapshot distributions available on a search path
  631. Any distributions found on `search_path` are added to the environment.
  632. `search_path` should be a sequence of ``sys.path`` items. If not
  633. supplied, ``sys.path`` is used.
  634. `platform` is an optional string specifying the name of the platform
  635. that platform-specific distributions must be compatible with. If
  636. unspecified, it defaults to the current platform. `python` is an
  637. optional string naming the desired version of Python (e.g. ``'3.3'``);
  638. it defaults to the current version.
  639. You may explicitly set `platform` (and/or `python`) to ``None`` if you
  640. wish to map *all* distributions, not just those compatible with the
  641. running platform or Python version.
  642. """
  643. self._distmap = {}
  644. self.platform = platform
  645. self.python = python
  646. self.scan(search_path)
  647. def can_add(self, dist):
  648. """Is distribution `dist` acceptable for this environment?
  649. The distribution must match the platform and python version
  650. requirements specified when this environment was created, or False
  651. is returned.
  652. """
  653. return (self.python is None or dist.py_version is None
  654. or dist.py_version==self.python) \
  655. and compatible_platforms(dist.platform, self.platform)
  656. def remove(self, dist):
  657. """Remove `dist` from the environment"""
  658. self._distmap[dist.key].remove(dist)
  659. def scan(self, search_path=None):
  660. """Scan `search_path` for distributions usable in this environment
  661. Any distributions found are added to the environment.
  662. `search_path` should be a sequence of ``sys.path`` items. If not
  663. supplied, ``sys.path`` is used. Only distributions conforming to
  664. the platform/python version defined at initialization are added.
  665. """
  666. if search_path is None:
  667. search_path = sys.path
  668. for item in search_path:
  669. for dist in find_distributions(item):
  670. self.add(dist)
  671. def __getitem__(self, project_name):
  672. """Return a newest-to-oldest list of distributions for `project_name`
  673. Uses case-insensitive `project_name` comparison, assuming all the
  674. project's distributions use their project's name converted to all
  675. lowercase as their key.
  676. """
  677. distribution_key = project_name.lower()
  678. return self._distmap.get(distribution_key, [])
  679. def add(self, dist):
  680. """Add `dist` if we ``can_add()`` it and it has not already been added
  681. """
  682. if self.can_add(dist) and dist.has_version():
  683. dists = self._distmap.setdefault(dist.key, [])
  684. if dist not in dists:
  685. dists.append(dist)
  686. dists.sort(key=operator.attrgetter('hashcmp'), reverse=True)
  687. def best_match(self, req, working_set, installer=None):
  688. """Find distribution best matching `req` and usable on `working_set`
  689. This calls the ``find(req)`` method of the `working_set` to see if a
  690. suitable distribution is already active. (This may raise
  691. ``VersionConflict`` if an unsuitable version of the project is already
  692. active in the specified `working_set`.) If a suitable distribution
  693. isn't active, this method returns the newest distribution in the
  694. environment that meets the ``Requirement`` in `req`. If no suitable
  695. distribution is found, and `installer` is supplied, then the result of
  696. calling the environment's ``obtain(req, installer)`` method will be
  697. returned.
  698. """
  699. dist = working_set.find(req)
  700. if dist is not None:
  701. return dist
  702. for dist in self[req.key]:
  703. if dist in req:
  704. return dist
  705. # try to download/install
  706. return self.obtain(req, installer)
  707. def obtain(self, requirement, installer=None):
  708. """Obtain a distribution matching `requirement` (e.g. via download)
  709. Obtain a distro that matches requirement (e.g. via download). In the
  710. base ``Environment`` class, this routine just returns
  711. ``installer(requirement)``, unless `installer` is None, in which case
  712. None is returned instead. This method is a hook that allows subclasses
  713. to attempt other ways of obtaining a distribution before falling back
  714. to the `installer` argument."""
  715. if installer is not None:
  716. return installer(requirement)
  717. def __iter__(self):
  718. """Yield the unique project names of the available distributions"""
  719. for key in self._distmap.keys():
  720. if self[key]: yield key
  721. def __iadd__(self, other):
  722. """In-place addition of a distribution or environment"""
  723. if isinstance(other, Distribution):
  724. self.add(other)
  725. elif isinstance(other, Environment):
  726. for project in other:
  727. for dist in other[project]:
  728. self.add(dist)
  729. else:
  730. raise TypeError("Can't add %r to environment" % (other,))
  731. return self
  732. def __add__(self, other):
  733. """Add an environment or distribution to an environment"""
  734. new = self.__class__([], platform=None, python=None)
  735. for env in self, other:
  736. new += env
  737. return new
  738. # XXX backward compatibility
  739. AvailableDistributions = Environment
  740. class ExtractionError(RuntimeError):
  741. """An error occurred extracting a resource
  742. The following attributes are available from instances of this exception:
  743. manager
  744. The resource manager that raised this exception
  745. cache_path
  746. The base directory for resource extraction
  747. original_error
  748. The exception instance that caused extraction to fail
  749. """
  750. class ResourceManager:
  751. """Manage resource extraction and packages"""
  752. extraction_path = None
  753. def __init__(self):
  754. self.cached_files = {}
  755. def resource_exists(self, package_or_requirement, resource_name):
  756. """Does the named resource exist?"""
  757. return get_provider(package_or_requirement).has_resource(resource_name)
  758. def resource_isdir(self, package_or_requirement, resource_name):
  759. """Is the named resource an existing directory?"""
  760. return get_provider(package_or_requirement).resource_isdir(
  761. resource_name
  762. )
  763. def resource_filename(self, package_or_requirement, resource_name):
  764. """Return a true filesystem path for specified resource"""
  765. return get_provider(package_or_requirement).get_resource_filename(
  766. self, resource_name
  767. )
  768. def resource_stream(self, package_or_requirement, resource_name):
  769. """Return a readable file-like object for specified resource"""
  770. return get_provider(package_or_requirement).get_resource_stream(
  771. self, resource_name
  772. )
  773. def resource_string(self, package_or_requirement, resource_name):
  774. """Return specified resource as a string"""
  775. return get_provider(package_or_requirement).get_resource_string(
  776. self, resource_name
  777. )
  778. def resource_listdir(self, package_or_requirement, resource_name):
  779. """List the contents of the named resource directory"""
  780. return get_provider(package_or_requirement).resource_listdir(
  781. resource_name
  782. )
  783. def extraction_error(self):
  784. """Give an error message for problems extracting file(s)"""
  785. old_exc = sys.exc_info()[1]
  786. cache_path = self.extraction_path or get_default_cache()
  787. err = ExtractionError("""Can't extract file(s) to egg cache
  788. The following error occurred while trying to extract file(s) to the Python egg
  789. cache:
  790. %s
  791. The Python egg cache directory is currently set to:
  792. %s
  793. Perhaps your account does not have write access to this directory? You can
  794. change the cache directory by setting the PYTHON_EGG_CACHE environment
  795. variable to point to an accessible directory.
  796. """ % (old_exc, cache_path)
  797. )
  798. err.manager = self
  799. err.cache_path = cache_path
  800. err.original_error = old_exc
  801. raise err
  802. def get_cache_path(self, archive_name, names=()):
  803. """Return absolute location in cache for `archive_name` and `names`
  804. The parent directory of the resulting path will be created if it does
  805. not already exist. `archive_name` should be the base filename of the
  806. enclosing egg (which may not be the name of the enclosing zipfile!),
  807. including its ".egg" extension. `names`, if provided, should be a
  808. sequence of path name parts "under" the egg's extraction location.
  809. This method should only be called by resource providers that need to
  810. obtain an extraction location, and only for names they intend to
  811. extract, as it tracks the generated names for possible cleanup later.
  812. """
  813. extract_path = self.extraction_path or get_default_cache()
  814. target_path = os.path.join(extract_path, archive_name+'-tmp', *names)
  815. try:
  816. _bypass_ensure_directory(target_path)
  817. except:
  818. self.extraction_error()
  819. self._warn_unsafe_extraction_path(extract_path)
  820. self.cached_files[target_path] = 1
  821. return target_path
  822. @staticmethod
  823. def _warn_unsafe_extraction_path(path):
  824. """
  825. If the default extraction path is overridden and set to an insecure
  826. location, such as /tmp, it opens up an opportunity for an attacker to
  827. replace an extracted file with an unauthorized payload. Warn the user
  828. if a known insecure location is used.
  829. See Distribute #375 for more details.
  830. """
  831. if os.name == 'nt' and not path.startswith(os.environ['windir']):
  832. # On Windows, permissions are generally restrictive by default
  833. # and temp directories are not writable by other users, so
  834. # bypass the warning.
  835. return
  836. mode = os.stat(path).st_mode
  837. if mode & stat.S_IWOTH or mode & stat.S_IWGRP:
  838. msg = ("%s is writable by group/others and vulnerable to attack "
  839. "when "
  840. "used with get_resource_filename. Consider a more secure "
  841. "location (set with .set_extraction_path or the "
  842. "PYTHON_EGG_CACHE environment variable)." % path)
  843. warnings.warn(msg, UserWarning)
  844. def postprocess(self, tempname, filename):
  845. """Perform any platform-specific postprocessing of `tempname`
  846. This is where Mac header rewrites should be done; other platforms don't
  847. have anything special they should do.
  848. Resource providers should call this method ONLY after successfully
  849. extracting a compressed resource. They must NOT call it on resources
  850. that are already in the filesystem.
  851. `tempname` is the current (temporary) name of the file, and `filename`
  852. is the name it will be renamed to by the caller after this routine
  853. returns.
  854. """
  855. if os.name == 'posix':
  856. # Make the resource executable
  857. mode = ((os.stat(tempname).st_mode) | 0o555) & 0o7777
  858. os.chmod(tempname, mode)
  859. def set_extraction_path(self, path):
  860. """Set the base path where resources will be extracted to, if needed.
  861. If you do not call this routine before any extractions take place, the
  862. path defaults to the return value of ``get_default_cache()``. (Which
  863. is based on the ``PYTHON_EGG_CACHE`` environment variable, with various
  864. platform-specific fallbacks. See that routine's documentation for more
  865. details.)
  866. Resources are extracted to subdirectories of this path based upon
  867. information given by the ``IResourceProvider``. You may set this to a
  868. temporary directory, but then you must call ``cleanup_resources()`` to
  869. delete the extracted files when done. There is no guarantee that
  870. ``cleanup_resources()`` will be able to remove all extracted files.
  871. (Note: you may not change the extraction path for a given resource
  872. manager once resources have been extracted, unless you first call
  873. ``cleanup_resources()``.)
  874. """
  875. if self.cached_files:
  876. raise ValueError(
  877. "Can't change extraction path, files already extracted"
  878. )
  879. self.extraction_path = path
  880. def cleanup_resources(self, force=False):
  881. """
  882. Delete all extracted resource files and directories, returning a list
  883. of the file and directory names that could not be successfully removed.
  884. This function does not have any concurrency protection, so it should
  885. generally only be called when the extraction path is a temporary
  886. directory exclusive to a single process. This method is not
  887. automatically called; you must call it explicitly or register it as an
  888. ``atexit`` function if you wish to ensure cleanup of a temporary
  889. directory used for extractions.
  890. """
  891. # XXX
  892. def get_default_cache():
  893. """Determine the default cache location
  894. This returns the ``PYTHON_EGG_CACHE`` environment variable, if set.
  895. Otherwise, on Windows, it returns a "Python-Eggs" subdirectory of the
  896. "Application Data" directory. On all other systems, it's "~/.python-eggs".
  897. """
  898. try:
  899. return os.environ['PYTHON_EGG_CACHE']
  900. except KeyError:
  901. pass
  902. if os.name!='nt':
  903. return os.path.expanduser('~/.python-eggs')
  904. # XXX this may be locale-specific!
  905. app_data = 'Application Data'
  906. app_homes = [
  907. # best option, should be locale-safe
  908. (('APPDATA',), None),
  909. (('USERPROFILE',), app_data),
  910. (('HOMEDRIVE','HOMEPATH'), app_data),
  911. (('HOMEPATH',), app_data),
  912. (('HOME',), None),
  913. # 95/98/ME
  914. (('WINDIR',), app_data),
  915. ]
  916. for keys, subdir in app_homes:
  917. dirname = ''
  918. for key in keys:
  919. if key in os.environ:
  920. dirname = os.path.join(dirname, os.environ[key])
  921. else:
  922. break
  923. else:
  924. if subdir:
  925. dirname = os.path.join(dirname, subdir)
  926. return os.path.join(dirname, 'Python-Eggs')
  927. else:
  928. raise RuntimeError(
  929. "Please set the PYTHON_EGG_CACHE enviroment variable"
  930. )
  931. def safe_name(name):
  932. """Convert an arbitrary string to a standard distribution name
  933. Any runs of non-alphanumeric/. characters are replaced with a single '-'.
  934. """
  935. return re.sub('[^A-Za-z0-9.]+', '-', name)
  936. def safe_version(version):
  937. """Convert an arbitrary string to a standard version string
  938. Spaces become dots, and all other non-alphanumeric characters become
  939. dashes, with runs of multiple dashes condensed to a single dash.
  940. """
  941. version = version.replace(' ','.')
  942. return re.sub('[^A-Za-z0-9.]+', '-', version)
  943. def safe_extra(extra):
  944. """Convert an arbitrary string to a standard 'extra' name
  945. Any runs of non-alphanumeric characters are replaced with a single '_',
  946. and the result is always lowercased.
  947. """
  948. return re.sub('[^A-Za-z0-9.]+', '_', extra).lower()
  949. def to_filename(name):
  950. """Convert a project or version name to its filename-escaped form
  951. Any '-' characters are currently replaced with '_'.
  952. """
  953. return name.replace('-','_')
  954. class MarkerEvaluation(object):
  955. values = {
  956. 'os_name': lambda: os.name,
  957. 'sys_platform': lambda: sys.platform,
  958. 'python_full_version': platform.python_version,
  959. 'python_version': lambda: platform.python_version()[:3],
  960. 'platform_version': platform.version,
  961. 'platform_machine': platform.machine,
  962. 'python_implementation': platform.python_implementation,
  963. }
  964. @classmethod
  965. def is_invalid_marker(cls, text):
  966. """
  967. Validate text as a PEP 426 environment marker; return an exception
  968. if invalid or False otherwise.
  969. """
  970. try:
  971. cls.evaluate_marker(text)
  972. except SyntaxError:
  973. return cls.normalize_exception(sys.exc_info()[1])
  974. return False
  975. @staticmethod
  976. def normalize_exception(exc):
  977. """
  978. Given a SyntaxError from a marker evaluation, normalize the error
  979. message:
  980. - Remove indications of filename and line number.
  981. - Replace platform-specific error messages with standard error
  982. messages.
  983. """
  984. subs = {
  985. 'unexpected EOF while parsing': 'invalid syntax',
  986. 'parenthesis is never closed': 'invalid syntax',
  987. }
  988. exc.filename = None
  989. exc.lineno = None
  990. exc.msg = subs.get(exc.msg, exc.msg)
  991. return exc
  992. @classmethod
  993. def and_test(cls, nodelist):
  994. # MUST NOT short-circuit evaluation, or invalid syntax can be skipped!
  995. items = [
  996. cls.interpret(nodelist[i])
  997. for i in range(1, len(nodelist), 2)
  998. ]
  999. return functools.reduce(operator.and_, items)
  1000. @classmethod
  1001. def test(cls, nodelist):
  1002. # MUST NOT short-circuit evaluation, or invalid syntax can be skipped!
  1003. items = [
  1004. cls.interpret(nodelist[i])
  1005. for i in range(1, len(nodelist), 2)
  1006. ]
  1007. return functools.reduce(operator.or_, items)
  1008. @classmethod
  1009. def atom(cls, nodelist):
  1010. t = nodelist[1][0]
  1011. if t == token.LPAR:
  1012. if nodelist[2][0] == token.RPAR:
  1013. raise SyntaxError("Empty parentheses")
  1014. return cls.interpret(nodelist[2])
  1015. msg = "Language feature not supported in environment markers"
  1016. raise SyntaxError(msg)
  1017. @classmethod
  1018. def comparison(cls, nodelist):
  1019. if len(nodelist) > 4:
  1020. msg = "Chained comparison not allowed in environment markers"
  1021. raise SyntaxError(msg)
  1022. comp = nodelist[2][1]
  1023. cop = comp[1]
  1024. if comp[0] == token.NAME:
  1025. if len(nodelist[2]) == 3:
  1026. if cop == 'not':
  1027. cop = 'not in'
  1028. else:
  1029. cop = 'is not'
  1030. try:
  1031. cop = cls.get_op(cop)
  1032. except KeyError:
  1033. msg = repr(cop) + " operator not allowed in environment markers"
  1034. raise SyntaxError(msg)
  1035. return cop(cls.evaluate(nodelist[1]), cls.evaluate(nodelist[3]))
  1036. @classmethod
  1037. def get_op(cls, op):
  1038. ops = {
  1039. symbol.test: cls.test,
  1040. symbol.and_test: cls.and_test,
  1041. symbol.atom: cls.atom,
  1042. symbol.comparison: cls.comparison,
  1043. 'not in': lambda x, y: x not in y,
  1044. 'in': lambda x, y: x in y,
  1045. '==': operator.eq,
  1046. '!=': operator.ne,
  1047. }
  1048. if hasattr(symbol, 'or_test'):
  1049. ops[symbol.or_test] = cls.test
  1050. return ops[op]
  1051. @classmethod
  1052. def evaluate_marker(cls, text, extra=None):
  1053. """
  1054. Evaluate a PEP 426 environment marker on CPython 2.4+.
  1055. Return a boolean indicating the marker result in this environment.
  1056. Raise SyntaxError if marker is invalid.
  1057. This implementation uses the 'parser' module, which is not implemented
  1058. on
  1059. Jython and has been superseded by the 'ast' module in Python 2.6 and
  1060. later.
  1061. """
  1062. return cls.interpret(parser.expr(text).totuple(1)[1])
  1063. @classmethod
  1064. def _markerlib_evaluate(cls, text):
  1065. """
  1066. Evaluate a PEP 426 environment marker using markerlib.
  1067. Return a boolean indicating the marker result in this environment.
  1068. Raise SyntaxError if marker is invalid.
  1069. """
  1070. import _markerlib
  1071. # markerlib implements Metadata 1.2 (PEP 345) environment markers.
  1072. # Translate the variables to Metadata 2.0 (PEP 426).
  1073. env = _markerlib.default_environment()
  1074. for key in env.keys():
  1075. new_key = key.replace('.', '_')
  1076. env[new_key] = env.pop(key)
  1077. try:
  1078. result = _markerlib.interpret(text, env)
  1079. except NameError:
  1080. e = sys.exc_info()[1]
  1081. raise SyntaxError(e.args[0])
  1082. return result
  1083. if 'parser' not in globals():
  1084. # Fall back to less-complete _markerlib implementation if 'parser' module
  1085. # is not available.
  1086. evaluate_marker = _markerlib_evaluate
  1087. @classmethod
  1088. def interpret(cls, nodelist):
  1089. while len(nodelist)==2: nodelist = nodelist[1]
  1090. try:
  1091. op = cls.get_op(nodelist[0])
  1092. except KeyError:
  1093. raise SyntaxError("Comparison or logical expression expected")
  1094. return op(nodelist)
  1095. @classmethod
  1096. def evaluate(cls, nodelist):
  1097. while len(nodelist)==2: nodelist = nodelist[1]
  1098. kind = nodelist[0]
  1099. name = nodelist[1]
  1100. if kind==token.NAME:
  1101. try:
  1102. op = cls.values[name]
  1103. except KeyError:
  1104. raise SyntaxError("Unknown name %r" % name)
  1105. return op()
  1106. if kind==token.STRING:
  1107. s = nodelist[1]
  1108. if not cls._safe_string(s):
  1109. raise SyntaxError(
  1110. "Only plain strings allowed in environment markers")
  1111. return s[1:-1]
  1112. msg = "Language feature not supported in environment markers"
  1113. raise SyntaxError(msg)
  1114. @staticmethod
  1115. def _safe_string(cand):
  1116. return (
  1117. cand[:1] in "'\"" and
  1118. not cand.startswith('"""') and
  1119. not cand.startswith("'''") and
  1120. '\\' not in cand
  1121. )
  1122. invalid_marker = MarkerEvaluation.is_invalid_marker
  1123. evaluate_marker = MarkerEvaluation.evaluate_marker
  1124. class NullProvider:
  1125. """Try to implement resources and metadata for arbitrary PEP 302 loaders"""
  1126. egg_name = None
  1127. egg_info = None
  1128. loader = None
  1129. def __init__(self, module):
  1130. self.loader = getattr(module, '__loader__', None)
  1131. self.module_path = os.path.dirname(getattr(module, '__file__', ''))
  1132. def get_resource_filename(self, manager, resource_name):
  1133. return self._fn(self.module_path, resource_name)
  1134. def get_resource_stream(self, manager, resource_name):
  1135. return BytesIO(self.get_resource_string(manager, resource_name))
  1136. def get_resource_string(self, manager, resource_name):
  1137. return self._get(self._fn(self.module_path, resource_name))
  1138. def has_resource(self, resource_name):
  1139. return self._has(self._fn(self.module_path, resource_name))
  1140. def has_metadata(self, name):
  1141. return self.egg_info and self._has(self._fn(self.egg_info, name))
  1142. if sys.version_info <= (3,):
  1143. def get_metadata(self, name):
  1144. if not self.egg_info:
  1145. return ""
  1146. return self._get(self._fn(self.egg_info, name))
  1147. else:
  1148. def get_metadata(self, name):
  1149. if not self.egg_info:
  1150. return ""
  1151. return self._get(self._fn(self.egg_info, name)).decode("utf-8")
  1152. def get_metadata_lines(self, name):
  1153. return yield_lines(self.get_metadata(name))
  1154. def resource_isdir(self, resource_name):
  1155. return self._isdir(self._fn(self.module_path, resource_name))
  1156. def metadata_isdir(self, name):
  1157. return self.egg_info and self._isdir(self._fn(self.egg_info, name))
  1158. def resource_listdir(self, resource_name):
  1159. return self._listdir(self._fn(self.module_path, resource_name))
  1160. def metadata_listdir(self, name):
  1161. if self.egg_info:
  1162. return self._listdir(self._fn(self.egg_info, name))
  1163. return []
  1164. def run_script(self, script_name, namespace):
  1165. script = 'scripts/'+script_name
  1166. if not self.has_metadata(script):
  1167. raise ResolutionError("No script named %r" % script_name)
  1168. script_text = self.get_metadata(script).replace('\r\n', '\n')
  1169. script_text = script_text.replace('\r', '\n')
  1170. script_filename = self._fn(self.egg_info, script)
  1171. namespace['__file__'] = script_filename
  1172. if os.path.exists(script_filename):
  1173. execfile(script_filename, namespace, namespace)
  1174. else:
  1175. from linecache import cache
  1176. cache[script_filename] = (
  1177. len(script_text), 0, script_text.split('\n'), script_filename
  1178. )
  1179. script_code = compile(script_text, script_filename,'exec')
  1180. exec(script_code, namespace, namespace)
  1181. def _has(self, path):
  1182. raise NotImplementedError(
  1183. "Can't perform this operation for unregistered loader type"
  1184. )
  1185. def _isdir(self, path):
  1186. raise NotImplementedError(
  1187. "Can't perform this operation for unregistered loader type"
  1188. )
  1189. def _listdir(self, path):
  1190. raise NotImplementedError(
  1191. "Can't perform this operation for unregistered loader type"
  1192. )
  1193. def _fn(self, base, resource_name):
  1194. if resource_name:
  1195. return os.path.join(base, *resource_name.split('/'))
  1196. return base
  1197. def _get(self, path):
  1198. if hasattr(self.loader, 'get_data'):
  1199. return self.loader.get_data(path)
  1200. raise NotImplementedError(
  1201. "Can't perform this operation for loaders without 'get_data()'"
  1202. )
  1203. register_loader_type(object, NullProvider)
  1204. class EggProvider(NullProvider):
  1205. """Provider based on a virtual filesystem"""
  1206. def __init__(self, module):
  1207. NullProvider.__init__(self, module)
  1208. self._setup_prefix()
  1209. def _setup_prefix(self):
  1210. # we assume here that our metadata may be nested inside a "basket"
  1211. # of multiple eggs; that's why we use module_path instead of .archive
  1212. path = self.module_path
  1213. old = None
  1214. while path!=old:
  1215. if path.lower().endswith('.egg'):
  1216. self.egg_name = os.path.basename(path)
  1217. self.egg_info = os.path.join(path, 'EGG-INFO')
  1218. self.egg_root = path
  1219. break
  1220. old = path
  1221. path, base = os.path.split(path)
  1222. class DefaultProvider(EggProvider):
  1223. """Provides access to package resources in the filesystem"""
  1224. def _has(self, path):
  1225. return os.path.exists(path)
  1226. def _isdir(self, path):
  1227. return os.path.isdir(path)
  1228. def _listdir(self, path):
  1229. return os.listdir(path)
  1230. def get_resource_stream(self, manager, resource_name):
  1231. return open(self._fn(self.module_path, resource_name), 'rb')
  1232. def _get(self, path):
  1233. stream = open(path, 'rb')
  1234. try:
  1235. return stream.read()
  1236. finally:
  1237. stream.close()
  1238. register_loader_type(type(None), DefaultProvider)
  1239. if importlib_bootstrap is not None:
  1240. register_loader_type(importlib_bootstrap.SourceFileLoader, DefaultProvider)
  1241. class EmptyProvider(NullProvider):
  1242. """Provider that returns nothing for all requests"""
  1243. _isdir = _has = lambda self, path: False
  1244. _get = lambda self, path: ''
  1245. _listdir = lambda self, path: []
  1246. module_path = None
  1247. def __init__(self):
  1248. pass
  1249. empty_provider = EmptyProvider()
  1250. def build_zipmanifest(path):
  1251. """
  1252. This builds a similar dictionary to the zipimport directory
  1253. caches. However instead of tuples, ZipInfo objects are stored.
  1254. The translation of the tuple is as follows:
  1255. * [0] - zipinfo.filename on stock pythons this needs "/" --> os.sep
  1256. on pypy it is the same (one reason why distribute did work
  1257. in some cases on pypy and win32).
  1258. * [1] - zipinfo.compress_type
  1259. * [2] - zipinfo.compress_size
  1260. * [3] - zipinfo.file_size
  1261. * [4] - len(utf-8 encoding of filename) if zipinfo & 0x800
  1262. len(ascii encoding of filename) otherwise
  1263. * [5] - (zipinfo.date_time[0] - 1980) << 9 |
  1264. zipinfo.date_time[1] << 5 | zipinfo.date_time[2]
  1265. * [6] - (zipinfo.date_time[3] - 1980) << 11 |
  1266. zipinfo.date_time[4] << 5 | (zipinfo.date_time[5] // 2)
  1267. * [7] - zipinfo.CRC
  1268. """
  1269. zipinfo = dict()
  1270. zfile = zipfile.ZipFile(path)
  1271. #Got ZipFile has not __exit__ on python 3.1
  1272. try:
  1273. for zitem in zfile.namelist():
  1274. zpath = zitem.replace('/', os.sep)
  1275. zipinfo[zpath] = zfile.getinfo(zitem)
  1276. assert zipinfo[zpath] is not None
  1277. finally:
  1278. zfile.close()
  1279. return zipinfo
  1280. class ZipProvider(EggProvider):
  1281. """Resource support for zips and eggs"""
  1282. eagers = None
  1283. def __init__(self, module):
  1284. EggProvider.__init__(self, module)
  1285. self.zipinfo = build_zipmanifest(self.loader.archive)
  1286. self.zip_pre = self.loader.archive+os.sep
  1287. def _zipinfo_name(self, fspath):
  1288. # Convert a virtual filename (full path to file) into a zipfile subpath
  1289. # usable with the zipimport directory cache for our target archive
  1290. if fspath.startswith(self.zip_pre):
  1291. return fspath[len(self.zip_pre):]
  1292. raise AssertionError(
  1293. "%s is not a subpath of %s" % (fspath, self.zip_pre)
  1294. )
  1295. def _parts(self, zip_path):
  1296. # Convert a zipfile subpath into an egg-relative path part list
  1297. fspath = self.zip_pre+zip_path # pseudo-fs path
  1298. if fspath.startswith(self.egg_root+os.sep):
  1299. return fspath[len(self.egg_root)+1:].split(os.sep)
  1300. raise AssertionError(
  1301. "%s is not a subpath of %s" % (fspath, self.egg_root)
  1302. )
  1303. def get_resource_filename(self, manager, resource_name):
  1304. if not self.egg_name:
  1305. raise NotImplementedError(
  1306. "resource_filename() only supported for .egg, not .zip"
  1307. )
  1308. # no need to lock for extraction, since we use temp names
  1309. zip_path = self._resource_to_zip(resource_name)
  1310. eagers = self._get_eager_resources()
  1311. if '/'.join(self._parts(zip_path)) in eagers:
  1312. for name in eagers:
  1313. self._extract_resource(manager, self._eager_to_zip(name))
  1314. return self._extract_resource(manager, zip_path)
  1315. @staticmethod
  1316. def _get_date_and_size(zip_stat):
  1317. size = zip_stat.file_size
  1318. # ymdhms+wday, yday, dst
  1319. date_time = zip_stat.date_time + (0, 0, -1)
  1320. # 1980 offset already done
  1321. timestamp = time.mktime(date_time)
  1322. return timestamp, size
  1323. def _extract_resource(self, manager, zip_path):
  1324. if zip_path in self._index():
  1325. for name in self._index()[zip_path]:
  1326. last = self._extract_resource(
  1327. manager, os.path.join(zip_path, name)
  1328. )
  1329. # return the extracted directory name
  1330. return os.path.dirname(last)
  1331. timestamp, size = self._get_date_and_size(self.zipinfo[zip_path])
  1332. if not WRITE_SUPPORT:
  1333. raise IOError('"os.rename" and "os.unlink" are not supported '
  1334. 'on this platform')
  1335. try:
  1336. real_path = manager.get_cache_path(
  1337. self.egg_name, self._parts(zip_path)
  1338. )
  1339. if self._is_current(real_path, zip_path):
  1340. return real_path
  1341. outf, tmpnam = _mkstemp(".$extract", dir=os.path.dirname(real_path))
  1342. os.write(outf, self.loader.get_data(zip_path))
  1343. os.close(outf)
  1344. utime(tmpnam, (timestamp, timestamp))
  1345. manager.postprocess(tmpnam, real_path)
  1346. try:
  1347. rename(tmpnam, real_path)
  1348. except os.error:
  1349. if os.path.isfile(real_path):
  1350. if self._is_current(real_path, zip_path):
  1351. # the file became current since it was checked above,
  1352. # so proceed.
  1353. return real_path
  1354. # Windows, del old file and retry
  1355. elif os.name=='nt':
  1356. unlink(real_path)
  1357. rename(tmpnam, real_path)
  1358. return real_path
  1359. raise
  1360. except os.error:
  1361. # report a user-friendly error
  1362. manager.extraction_error()
  1363. return real_path
  1364. def _is_current(self, file_path, zip_path):
  1365. """
  1366. Return True if the file_path is current for this zip_path
  1367. """
  1368. timestamp, size = self._get_date_and_size(self.zipinfo[zip_path])
  1369. if not os.path.isfile(file_path):
  1370. return False
  1371. stat = os.stat(file_path)
  1372. if stat.st_size!=size or stat.st_mtime!=timestamp:
  1373. return False
  1374. # check that the contents match
  1375. zip_contents = self.loader.get_data(zip_path)
  1376. f = open(file_path, 'rb')
  1377. file_contents = f.read()
  1378. f.close()
  1379. return zip_contents == file_contents
  1380. def _get_eager_resources(self):
  1381. if self.eagers is None:
  1382. eagers = []
  1383. for name in ('native_libs.txt', 'eager_resources.txt'):
  1384. if self.has_metadata(name):
  1385. eagers.extend(self.get_metadata_lines(name))
  1386. self.eagers = eagers
  1387. return self.eagers
  1388. def _index(self):
  1389. try:
  1390. return self._dirindex
  1391. except AttributeError:
  1392. ind = {}
  1393. for path in self.zipinfo:
  1394. parts = path.split(os.sep)
  1395. while parts:
  1396. parent = os.sep.join(parts[:-1])
  1397. if parent in ind:
  1398. ind[parent].append(parts[-1])
  1399. break
  1400. else:
  1401. ind[parent] = [parts.pop()]
  1402. self._dirindex = ind
  1403. return ind
  1404. def _has(self, fspath):
  1405. zip_path = self._zipinfo_name(fspath)
  1406. return zip_path in self.zipinfo or zip_path in self._index()
  1407. def _isdir(self, fspath):
  1408. return self._zipinfo_name(fspath) in self._index()
  1409. def _listdir(self, fspath):
  1410. return list(self._index().get(self._zipinfo_name(fspath), ()))
  1411. def _eager_to_zip(self, resource_name):
  1412. return self._zipinfo_name(self._fn(self.egg_root, resource_name))
  1413. def _resource_to_zip(self, resource_name):
  1414. return self._zipinfo_name(self._fn(self.module_path, resource_name))
  1415. register_loader_type(zipimport.zipimporter, ZipProvider)
  1416. class FileMetadata(EmptyProvider):
  1417. """Metadata handler for standalone PKG-INFO files
  1418. Usage::
  1419. metadata = FileMetadata("/path/to/PKG-INFO")
  1420. This provider rejects all data and metadata requests except for PKG-INFO,
  1421. which is treated as existing, and will be the contents of the file at
  1422. the provided location.
  1423. """
  1424. def __init__(self, path):
  1425. self.path = path
  1426. def has_metadata(self, name):
  1427. return name=='PKG-INFO'
  1428. def get_metadata(self, name):
  1429. if name=='PKG-INFO':
  1430. f = open(self.path,'rU')
  1431. metadata = f.read()
  1432. f.close()
  1433. return metadata
  1434. raise KeyError("No metadata except PKG-INFO is available")
  1435. def get_metadata_lines(self, name):
  1436. return yield_lines(self.get_metadata(name))
  1437. class PathMetadata(DefaultProvider):
  1438. """Metadata provider for egg directories
  1439. Usage::
  1440. # Development eggs:
  1441. egg_info = "/path/to/PackageName.egg-info"
  1442. base_dir = os.path.dirname(egg_info)
  1443. metadata = PathMetadata(base_dir, egg_info)
  1444. dist_name = os.path.splitext(os.path.basename(egg_info))[0]
  1445. dist = Distribution(basedir, project_name=dist_name, metadata=metadata)
  1446. # Unpacked egg directories:
  1447. egg_path = "/path/to/PackageName-ver-pyver-etc.egg"
  1448. metadata = PathMetadata(egg_path, os.path.join(egg_path,'EGG-INFO'))
  1449. dist = Distribution.from_filename(egg_path, metadata=metadata)
  1450. """
  1451. def __init__(self, path, egg_info):
  1452. self.module_path = path
  1453. self.egg_info = egg_info
  1454. class EggMetadata(ZipProvider):
  1455. """Metadata provider for .egg files"""
  1456. def __init__(self, importer):
  1457. """Create a metadata provider from a zipimporter"""
  1458. self.zipinfo = build_zipmanifest(importer.archive)
  1459. self.zip_pre = importer.archive+os.sep
  1460. self.loader = importer
  1461. if importer.prefix:
  1462. self.module_path = os.path.join(importer.archive, importer.prefix)
  1463. else:
  1464. self.module_path = importer.archive
  1465. self._setup_prefix()
  1466. _declare_state('dict', _distribution_finders = {})
  1467. def register_finder(importer_type, distribution_finder):
  1468. """Register `distribution_finder` to find distributions in sys.path items
  1469. `importer_type` is the type or class of a PEP 302 "Importer" (sys.path item
  1470. handler), and `distribution_finder` is a callable that, passed a path
  1471. item and the importer instance, yields ``Distribution`` instances found on
  1472. that path item. See ``pkg_resources.find_on_path`` for an example."""
  1473. _distribution_finders[importer_type] = distribution_finder
  1474. def find_distributions(path_item, only=False):
  1475. """Yield distributions accessible via `path_item`"""
  1476. importer = get_importer(path_item)
  1477. finder = _find_adapter(_distribution_finders, importer)
  1478. return finder(importer, path_item, only)
  1479. def find_eggs_in_zip(importer, path_item, only=False):
  1480. """
  1481. Find eggs in zip files; possibly multiple nested eggs.
  1482. """
  1483. if importer.archive.endswith('.whl'):
  1484. # wheels are not supported with this finder
  1485. # they don't have PKG-INFO metadata, and won't ever contain eggs
  1486. return
  1487. metadata = EggMetadata(importer)
  1488. if metadata.has_metadata('PKG-INFO'):
  1489. yield Distribution.from_filename(path_item, metadata=metadata)
  1490. if only:
  1491. # don't yield nested distros
  1492. return
  1493. for subitem in metadata.resource_listdir('/'):
  1494. if subitem.endswith('.egg'):
  1495. subpath = os.path.join(path_item, subitem)
  1496. for dist in find_eggs_in_zip(zipimport.zipimporter(subpath), subpath):
  1497. yield dist
  1498. register_finder(zipimport.zipimporter, find_eggs_in_zip)
  1499. def find_nothing(importer, path_item, only=False):
  1500. return ()
  1501. register_finder(object, find_nothing)
  1502. def find_on_path(importer, path_item, only=False):
  1503. """Yield distributions accessible on a sys.path directory"""
  1504. path_item = _normalize_cached(path_item)
  1505. if os.path.isdir(path_item) and os.access(path_item, os.R_OK):
  1506. if path_item.lower().endswith('.egg'):
  1507. # unpacked egg
  1508. yield Distribution.from_filename(
  1509. path_item, metadata=PathMetadata(
  1510. path_item, os.path.join(path_item,'EGG-INFO')
  1511. )
  1512. )
  1513. else:
  1514. # scan for .egg and .egg-info in directory
  1515. for entry in os.listdir(path_item):
  1516. lower = entry.lower()
  1517. if lower.endswith('.egg-info') or lower.endswith('.dist-info'):
  1518. fullpath = os.path.join(path_item, entry)
  1519. if os.path.isdir(fullpath):
  1520. # egg-info directory, allow getting metadata
  1521. metadata = PathMetadata(path_item, fullpath)
  1522. else:
  1523. metadata = FileMetadata(fullpath)
  1524. yield Distribution.from_location(
  1525. path_item, entry, metadata, precedence=DEVELOP_DIST
  1526. )
  1527. elif not only and lower.endswith('.egg'):
  1528. dists = find_distributions(os.path.join(path_item, entry))
  1529. for dist in dists:
  1530. yield dist
  1531. elif not only and lower.endswith('.egg-link'):
  1532. entry_file = open(os.path.join(path_item, entry))
  1533. try:
  1534. entry_lines = entry_file.readlines()
  1535. finally:
  1536. entry_file.close()
  1537. for line in entry_lines:
  1538. if not line.strip():
  1539. continue
  1540. path = os.path.join(path_item, line.rstrip())
  1541. dists = find_distributions(path)
  1542. for item in dists:
  1543. yield item
  1544. break
  1545. register_finder(pkgutil.ImpImporter, find_on_path)
  1546. if importlib_bootstrap is not None:
  1547. register_finder(importlib_bootstrap.FileFinder, find_on_path)
  1548. _declare_state('dict', _namespace_handlers={})
  1549. _declare_state('dict', _namespace_packages={})
  1550. def register_namespace_handler(importer_type, namespace_handler):
  1551. """Register `namespace_handler` to declare namespace packages
  1552. `importer_type` is the type or class of a PEP 302 "Importer" (sys.path item
  1553. handler), and `namespace_handler` is a callable like this::
  1554. def namespace_handler(importer, path_entry, moduleName, module):
  1555. # return a path_entry to use for child packages
  1556. Namespace handlers are only called if the importer object has already
  1557. agreed that it can handle the relevant path item, and they should only
  1558. return a subpath if the module __path__ does not already contain an
  1559. equivalent subpath. For an example namespace handler, see
  1560. ``pkg_resources.file_ns_handler``.
  1561. """
  1562. _namespace_handlers[importer_type] = namespace_handler
  1563. def _handle_ns(packageName, path_item):
  1564. """Ensure that named package includes a subpath of path_item (if needed)"""
  1565. importer = get_importer(path_item)
  1566. if importer is None:
  1567. return None
  1568. loader = importer.find_module(packageName)
  1569. if loader is None:
  1570. return None
  1571. module = sys.modules.get(packageName)
  1572. if module is None:
  1573. module = sys.modules[packageName] = imp.new_module(packageName)
  1574. module.__path__ = []
  1575. _set_parent_ns(packageName)
  1576. elif not hasattr(module,'__path__'):
  1577. raise TypeError("Not a package:", packageName)
  1578. handler = _find_adapter(_namespace_handlers, importer)
  1579. subpath = handler(importer, path_item, packageName, module)
  1580. if subpath is not None:
  1581. path = module.__path__
  1582. path.append(subpath)
  1583. loader.load_module(packageName)
  1584. for path_item in path:
  1585. if path_item not in module.__path__:
  1586. module.__path__.append(path_item)
  1587. return subpath
  1588. def declare_namespace(packageName):
  1589. """Declare that package 'packageName' is a namespace package"""
  1590. imp.acquire_lock()
  1591. try:
  1592. if packageName in _namespace_packages:
  1593. return
  1594. path, parent = sys.path, None
  1595. if '.' in packageName:
  1596. parent = '.'.join(packageName.split('.')[:-1])
  1597. declare_namespace(parent)
  1598. if parent not in _namespace_packages:
  1599. __import__(parent)
  1600. try:
  1601. path = sys.modules[parent].__path__
  1602. except AttributeError:
  1603. raise TypeError("Not a package:", parent)
  1604. # Track what packages are namespaces, so when new path items are added,
  1605. # they can be updated
  1606. _namespace_packages.setdefault(parent,[]).append(packageName)
  1607. _namespace_packages.setdefault(packageName,[])
  1608. for path_item in path:
  1609. # Ensure all the parent's path items are reflected in the child,
  1610. # if they apply
  1611. _handle_ns(packageName, path_item)
  1612. finally:
  1613. imp.release_lock()
  1614. def fixup_namespace_packages(path_item, parent=None):
  1615. """Ensure that previously-declared namespace packages include path_item"""
  1616. imp.acquire_lock()
  1617. try:
  1618. for package in _namespace_packages.get(parent,()):
  1619. subpath = _handle_ns(package, path_item)
  1620. if subpath: fixup_namespace_packages(subpath, package)
  1621. finally:
  1622. imp.release_lock()
  1623. def file_ns_handler(importer, path_item, packageName, module):
  1624. """Compute an ns-package subpath for a filesystem or zipfile importer"""
  1625. subpath = os.path.join(path_item, packageName.split('.')[-1])
  1626. normalized = _normalize_cached(subpath)
  1627. for item in module.__path__:
  1628. if _normalize_cached(item)==normalized:
  1629. break
  1630. else:
  1631. # Only return the path if it's not already there
  1632. return subpath
  1633. register_namespace_handler(pkgutil.ImpImporter, file_ns_handler)
  1634. register_namespace_handler(zipimport.zipimporter, file_ns_handler)
  1635. if importlib_bootstrap is not None:
  1636. register_namespace_handler(importlib_bootstrap.FileFinder, file_ns_handler)
  1637. def null_ns_handler(importer, path_item, packageName, module):
  1638. return None
  1639. register_namespace_handler(object, null_ns_handler)
  1640. def normalize_path(filename):
  1641. """Normalize a file/dir name for comparison purposes"""
  1642. return os.path.normcase(os.path.realpath(filename))
  1643. def _normalize_cached(filename, _cache={}):
  1644. try:
  1645. return _cache[filename]
  1646. except KeyError:
  1647. _cache[filename] = result = normalize_path(filename)
  1648. return result
  1649. def _set_parent_ns(packageName):
  1650. parts = packageName.split('.')
  1651. name = parts.pop()
  1652. if parts:
  1653. parent = '.'.join(parts)
  1654. setattr(sys.modules[parent], name, sys.modules[packageName])
  1655. def yield_lines(strs):
  1656. """Yield non-empty/non-comment lines of a ``basestring`` or sequence"""
  1657. if isinstance(strs, basestring):
  1658. for s in strs.splitlines():
  1659. s = s.strip()
  1660. # skip blank lines/comments
  1661. if s and not s.startswith('#'):
  1662. yield s
  1663. else:
  1664. for ss in strs:
  1665. for s in yield_lines(ss):
  1666. yield s
  1667. # whitespace and comment
  1668. LINE_END = re.compile(r"\s*(#.*)?$").match
  1669. # line continuation
  1670. CONTINUE = re.compile(r"\s*\\\s*(#.*)?$").match
  1671. # Distribution or extra
  1672. DISTRO = re.compile(r"\s*((\w|[-.])+)").match
  1673. # ver. info
  1674. VERSION = re.compile(r"\s*(<=?|>=?|==|!=)\s*((\w|[-.])+)").match
  1675. # comma between items
  1676. COMMA = re.compile(r"\s*,").match
  1677. OBRACKET = re.compile(r"\s*\[").match
  1678. CBRACKET = re.compile(r"\s*\]").match
  1679. MODULE = re.compile(r"\w+(\.\w+)*$").match
  1680. EGG_NAME = re.compile(
  1681. r"(?P<name>[^-]+)"
  1682. r"( -(?P<ver>[^-]+) (-py(?P<pyver>[^-]+) (-(?P<plat>.+))? )? )?",
  1683. re.VERBOSE | re.IGNORECASE
  1684. ).match
  1685. component_re = re.compile(r'(\d+ | [a-z]+ | \.| -)', re.VERBOSE)
  1686. replace = {'pre':'c', 'preview':'c','-':'final-','rc':'c','dev':'@'}.get
  1687. def _parse_version_parts(s):
  1688. for part in component_re.split(s):
  1689. part = replace(part, part)
  1690. if not part or part=='.':
  1691. continue
  1692. if part[:1] in '0123456789':
  1693. # pad for numeric comparison
  1694. yield part.zfill(8)
  1695. else:
  1696. yield '*'+part
  1697. # ensure that alpha/beta/candidate are before final
  1698. yield '*final'
  1699. def parse_version(s):
  1700. """Convert a version string to a chronologically-sortable key
  1701. This is a rough cross between distutils' StrictVersion and LooseVersion;
  1702. if you give it versions that would work with StrictVersion, then it behaves
  1703. the same; otherwise it acts like a slightly-smarter LooseVersion. It is
  1704. *possible* to create pathological version coding schemes that will fool
  1705. this parser, but they should be very rare in practice.
  1706. The returned value will be a tuple of strings. Numeric portions of the
  1707. version are padded to 8 digits so they will compare numerically, but
  1708. without relying on how numbers compare relative to strings. Dots are
  1709. dropped, but dashes are retained. Trailing zeros between alpha segments
  1710. or dashes are suppressed, so that e.g. "2.4.0" is considered the same as
  1711. "2.4". Alphanumeric parts are lower-cased.
  1712. The algorithm assumes that strings like "-" and any alpha string that
  1713. alphabetically follows "final" represents a "patch level". So, "2.4-1"
  1714. is assumed to be a branch or patch of "2.4", and therefore "2.4.1" is
  1715. considered newer than "2.4-1", which in turn is newer than "2.4".
  1716. Strings like "a", "b", "c", "alpha", "beta", "candidate" and so on (that
  1717. come before "final" alphabetically) are assumed to be pre-release versions,
  1718. so that the version "2.4" is considered newer than "2.4a1".
  1719. Finally, to handle miscellaneous cases, the strings "pre", "preview", and
  1720. "rc" are treated as if they were "c", i.e. as though they were release
  1721. candidates, and therefore are not as new as a version string that does not
  1722. contain them, and "dev" is replaced with an '@' so that it sorts lower than
  1723. than any other pre-release tag.
  1724. """
  1725. parts = []
  1726. for part in _parse_version_parts(s.lower()):
  1727. if part.startswith('*'):
  1728. # remove '-' before a prerelease tag
  1729. if part<'*final':
  1730. while parts and parts[-1]=='*final-': parts.pop()
  1731. # remove trailing zeros from each series of numeric parts
  1732. while parts and parts[-1]=='00000000':
  1733. parts.pop()
  1734. parts.append(part)
  1735. return tuple(parts)
  1736. class EntryPoint(object):
  1737. """Object representing an advertised importable object"""
  1738. def __init__(self, name, module_name, attrs=(), extras=(), dist=None):
  1739. if not MODULE(module_name):
  1740. raise ValueError("Invalid module name", module_name)
  1741. self.name = name
  1742. self.module_name = module_name
  1743. self.attrs = tuple(attrs)
  1744. self.extras = Requirement.parse(("x[%s]" % ','.join(extras))).extras
  1745. self.dist = dist
  1746. def __str__(self):
  1747. s = "%s = %s" % (self.name, self.module_name)
  1748. if self.attrs:
  1749. s += ':' + '.'.join(self.attrs)
  1750. if self.extras:
  1751. s += ' [%s]' % ','.join(self.extras)
  1752. return s
  1753. def __repr__(self):
  1754. return "EntryPoint.parse(%r)" % str(self)
  1755. def load(self, require=True, env=None, installer=None):
  1756. if require: self.require(env, installer)
  1757. entry = __import__(self.module_name, globals(), globals(),
  1758. ['__name__'])
  1759. for attr in self.attrs:
  1760. try:
  1761. entry = getattr(entry, attr)
  1762. except AttributeError:
  1763. raise ImportError("%r has no %r attribute" % (entry, attr))
  1764. return entry
  1765. def require(self, env=None, installer=None):
  1766. if self.extras and not self.dist:
  1767. raise UnknownExtra("Can't require() without a distribution", self)
  1768. reqs = self.dist.requires(self.extras)
  1769. items = working_set.resolve(reqs, env, installer)
  1770. list(map(working_set.add, items))
  1771. @classmethod
  1772. def parse(cls, src, dist=None):
  1773. """Parse a single entry point from string `src`
  1774. Entry point syntax follows the form::
  1775. name = some.module:some.attr [extra1, extra2]
  1776. The entry name and module name are required, but the ``:attrs`` and
  1777. ``[extras]`` parts are optional
  1778. """
  1779. try:
  1780. attrs = extras = ()
  1781. name, value = src.split('=',1)
  1782. if '[' in value:
  1783. value, extras = value.split('[',1)
  1784. req = Requirement.parse("x["+extras)
  1785. if req.specs: raise ValueError
  1786. extras = req.extras
  1787. if ':' in value:
  1788. value, attrs = value.split(':',1)
  1789. if not MODULE(attrs.rstrip()):
  1790. raise ValueError
  1791. attrs = attrs.rstrip().split('.')
  1792. except ValueError:
  1793. raise ValueError(
  1794. "EntryPoint must be in 'name=module:attrs [extras]' format",
  1795. src
  1796. )
  1797. else:
  1798. return cls(name.strip(), value.strip(), attrs, extras, dist)
  1799. @classmethod
  1800. def parse_group(cls, group, lines, dist=None):
  1801. """Parse an entry point group"""
  1802. if not MODULE(group):
  1803. raise ValueError("Invalid group name", group)
  1804. this = {}
  1805. for line in yield_lines(lines):
  1806. ep = cls.parse(line, dist)
  1807. if ep.name in this:
  1808. raise ValueError("Duplicate entry point", group, ep.name)
  1809. this[ep.name]=ep
  1810. return this
  1811. @classmethod
  1812. def parse_map(cls, data, dist=None):
  1813. """Parse a map of entry point groups"""
  1814. if isinstance(data, dict):
  1815. data = data.items()
  1816. else:
  1817. data = split_sections(data)
  1818. maps = {}
  1819. for group, lines in data:
  1820. if group is None:
  1821. if not lines:
  1822. continue
  1823. raise ValueError("Entry points must be listed in groups")
  1824. group = group.strip()
  1825. if group in maps:
  1826. raise ValueError("Duplicate group name", group)
  1827. maps[group] = cls.parse_group(group, lines, dist)
  1828. return maps
  1829. def _remove_md5_fragment(location):
  1830. if not location:
  1831. return ''
  1832. parsed = urlparse(location)
  1833. if parsed[-1].startswith('md5='):
  1834. return urlunparse(parsed[:-1] + ('',))
  1835. return location
  1836. class Distribution(object):
  1837. """Wrap an actual or potential sys.path entry w/metadata"""
  1838. PKG_INFO = 'PKG-INFO'
  1839. def __init__(self, location=None, metadata=None, project_name=None,
  1840. version=None, py_version=PY_MAJOR, platform=None,
  1841. precedence=EGG_DIST):
  1842. self.project_name = safe_name(project_name or 'Unknown')
  1843. if version is not None:
  1844. self._version = safe_version(version)
  1845. self.py_version = py_version
  1846. self.platform = platform
  1847. self.location = location
  1848. self.precedence = precedence
  1849. self._provider = metadata or empty_provider
  1850. @classmethod
  1851. def from_location(cls, location, basename, metadata=None,**kw):
  1852. project_name, version, py_version, platform = [None]*4
  1853. basename, ext = os.path.splitext(basename)
  1854. if ext.lower() in _distributionImpl:
  1855. # .dist-info gets much metadata differently
  1856. match = EGG_NAME(basename)
  1857. if match:
  1858. project_name, version, py_version, platform = match.group(
  1859. 'name','ver','pyver','plat'
  1860. )
  1861. cls = _distributionImpl[ext.lower()]
  1862. return cls(
  1863. location, metadata, project_name=project_name, version=version,
  1864. py_version=py_version, platform=platform, **kw
  1865. )
  1866. @property
  1867. def hashcmp(self):
  1868. return (
  1869. getattr(self, 'parsed_version', ()),
  1870. self.precedence,
  1871. self.key,
  1872. _remove_md5_fragment(self.location),
  1873. self.py_version,
  1874. self.platform,
  1875. )
  1876. def __hash__(self):
  1877. return hash(self.hashcmp)
  1878. def __lt__(self, other):
  1879. return self.hashcmp < other.hashcmp
  1880. def __le__(self, other):
  1881. return self.hashcmp <= other.hashcmp
  1882. def __gt__(self, other):
  1883. return self.hashcmp > other.hashcmp
  1884. def __ge__(self, other):
  1885. return self.hashcmp >= other.hashcmp
  1886. def __eq__(self, other):
  1887. if not isinstance(other, self.__class__):
  1888. # It's not a Distribution, so they are not equal
  1889. return False
  1890. return self.hashcmp == other.hashcmp
  1891. def __ne__(self, other):
  1892. return not self == other
  1893. # These properties have to be lazy so that we don't have to load any
  1894. # metadata until/unless it's actually needed. (i.e., some distributions
  1895. # may not know their name or version without loading PKG-INFO)
  1896. @property
  1897. def key(self):
  1898. try:
  1899. return self._key
  1900. except AttributeError:
  1901. self._key = key = self.project_name.lower()
  1902. return key
  1903. @property
  1904. def parsed_version(self):
  1905. try:
  1906. return self._parsed_version
  1907. except AttributeError:
  1908. self._parsed_version = pv = parse_version(self.version)
  1909. return pv
  1910. @property
  1911. def version(self):
  1912. try:
  1913. return self._version
  1914. except AttributeError:
  1915. for line in self._get_metadata(self.PKG_INFO):
  1916. if line.lower().startswith('version:'):
  1917. self._version = safe_version(line.split(':',1)[1].strip())
  1918. return self._version
  1919. else:
  1920. tmpl = "Missing 'Version:' header and/or %s file"
  1921. raise ValueError(tmpl % self.PKG_INFO, self)
  1922. @property
  1923. def _dep_map(self):
  1924. try:
  1925. return self.__dep_map
  1926. except AttributeError:
  1927. dm = self.__dep_map = {None: []}
  1928. for name in 'requires.txt', 'depends.txt':
  1929. for extra, reqs in split_sections(self._get_metadata(name)):
  1930. if extra:
  1931. if ':' in extra:
  1932. extra, marker = extra.split(':',1)
  1933. if invalid_marker(marker):
  1934. # XXX warn
  1935. reqs=[]
  1936. elif not evaluate_marker(marker):
  1937. reqs=[]
  1938. extra = safe_extra(extra) or None
  1939. dm.setdefault(extra,[]).extend(parse_requirements(reqs))
  1940. return dm
  1941. def requires(self, extras=()):
  1942. """List of Requirements needed for this distro if `extras` are used"""
  1943. dm = self._dep_map
  1944. deps = []
  1945. deps.extend(dm.get(None,()))
  1946. for ext in extras:
  1947. try:
  1948. deps.extend(dm[safe_extra(ext)])
  1949. except KeyError:
  1950. raise UnknownExtra(
  1951. "%s has no such extra feature %r" % (self, ext)
  1952. )
  1953. return deps
  1954. def _get_metadata(self, name):
  1955. if self.has_metadata(name):
  1956. for line in self.get_metadata_lines(name):
  1957. yield line
  1958. def activate(self, path=None):
  1959. """Ensure distribution is importable on `path` (default=sys.path)"""
  1960. if path is None: path = sys.path
  1961. self.insert_on(path)
  1962. if path is sys.path:
  1963. fixup_namespace_packages(self.location)
  1964. for pkg in self._get_metadata('namespace_packages.txt'):
  1965. if pkg in sys.modules:
  1966. declare_namespace(pkg)
  1967. def egg_name(self):
  1968. """Return what this distribution's standard .egg filename should be"""
  1969. filename = "%s-%s-py%s" % (
  1970. to_filename(self.project_name), to_filename(self.version),
  1971. self.py_version or PY_MAJOR
  1972. )
  1973. if self.platform:
  1974. filename += '-'+self.platform
  1975. return filename
  1976. def __repr__(self):
  1977. if self.location:
  1978. return "%s (%s)" % (self, self.location)
  1979. else:
  1980. return str(self)
  1981. def __str__(self):
  1982. try: version = getattr(self,'version',None)
  1983. except ValueError: version = None
  1984. version = version or "[unknown version]"
  1985. return "%s %s" % (self.project_name, version)
  1986. def __getattr__(self, attr):
  1987. """Delegate all unrecognized public attributes to .metadata provider"""
  1988. if attr.startswith('_'):
  1989. raise AttributeError(attr)
  1990. return getattr(self._provider, attr)
  1991. @classmethod
  1992. def from_filename(cls, filename, metadata=None, **kw):
  1993. return cls.from_location(
  1994. _normalize_cached(filename), os.path.basename(filename), metadata,
  1995. **kw
  1996. )
  1997. def as_requirement(self):
  1998. """Return a ``Requirement`` that matches this distribution exactly"""
  1999. return Requirement.parse('%s==%s' % (self.project_name, self.version))
  2000. def load_entry_point(self, group, name):
  2001. """Return the `name` entry point of `group` or raise ImportError"""
  2002. ep = self.get_entry_info(group, name)
  2003. if ep is None:
  2004. raise ImportError("Entry point %r not found" % ((group, name),))
  2005. return ep.load()
  2006. def get_entry_map(self, group=None):
  2007. """Return the entry point map for `group`, or the full entry map"""
  2008. try:
  2009. ep_map = self._ep_map
  2010. except AttributeError:
  2011. ep_map = self._ep_map = EntryPoint.parse_map(
  2012. self._get_metadata('entry_points.txt'), self
  2013. )
  2014. if group is not None:
  2015. return ep_map.get(group,{})
  2016. return ep_map
  2017. def get_entry_info(self, group, name):
  2018. """Return the EntryPoint object for `group`+`name`, or ``None``"""
  2019. return self.get_entry_map(group).get(name)
  2020. def insert_on(self, path, loc = None):
  2021. """Insert self.location in path before its nearest parent directory"""
  2022. loc = loc or self.location
  2023. if not loc:
  2024. return
  2025. nloc = _normalize_cached(loc)
  2026. bdir = os.path.dirname(nloc)
  2027. npath= [(p and _normalize_cached(p) or p) for p in path]
  2028. for p, item in enumerate(npath):
  2029. if item==nloc:
  2030. break
  2031. elif item==bdir and self.precedence==EGG_DIST:
  2032. # if it's an .egg, give it precedence over its directory
  2033. if path is sys.path:
  2034. self.check_version_conflict()
  2035. path.insert(p, loc)
  2036. npath.insert(p, nloc)
  2037. break
  2038. else:
  2039. if path is sys.path:
  2040. self.check_version_conflict()
  2041. path.append(loc)
  2042. return
  2043. # p is the spot where we found or inserted loc; now remove duplicates
  2044. while 1:
  2045. try:
  2046. np = npath.index(nloc, p+1)
  2047. except ValueError:
  2048. break
  2049. else:
  2050. del npath[np], path[np]
  2051. # ha!
  2052. p = np
  2053. return
  2054. def check_version_conflict(self):
  2055. if self.key=='setuptools':
  2056. # ignore the inevitable setuptools self-conflicts :(
  2057. return
  2058. nsp = dict.fromkeys(self._get_metadata('namespace_packages.txt'))
  2059. loc = normalize_path(self.location)
  2060. for modname in self._get_metadata('top_level.txt'):
  2061. if (modname not in sys.modules or modname in nsp
  2062. or modname in _namespace_packages):
  2063. continue
  2064. if modname in ('pkg_resources', 'setuptools', 'site'):
  2065. continue
  2066. fn = getattr(sys.modules[modname], '__file__', None)
  2067. if fn and (normalize_path(fn).startswith(loc) or
  2068. fn.startswith(self.location)):
  2069. continue
  2070. issue_warning(
  2071. "Module %s was already imported from %s, but %s is being added"
  2072. " to sys.path" % (modname, fn, self.location),
  2073. )
  2074. def has_version(self):
  2075. try:
  2076. self.version
  2077. except ValueError:
  2078. issue_warning("Unbuilt egg for "+repr(self))
  2079. return False
  2080. return True
  2081. def clone(self,**kw):
  2082. """Copy this distribution, substituting in any changed keyword args"""
  2083. for attr in (
  2084. 'project_name', 'version', 'py_version', 'platform', 'location',
  2085. 'precedence'
  2086. ):
  2087. kw.setdefault(attr, getattr(self, attr, None))
  2088. kw.setdefault('metadata', self._provider)
  2089. return self.__class__(**kw)
  2090. @property
  2091. def extras(self):
  2092. return [dep for dep in self._dep_map if dep]
  2093. class DistInfoDistribution(Distribution):
  2094. """Wrap an actual or potential sys.path entry w/metadata, .dist-info style"""
  2095. PKG_INFO = 'METADATA'
  2096. EQEQ = re.compile(r"([\(,])\s*(\d.*?)\s*([,\)])")
  2097. @property
  2098. def _parsed_pkg_info(self):
  2099. """Parse and cache metadata"""
  2100. try:
  2101. return self._pkg_info
  2102. except AttributeError:
  2103. from email.parser import Parser
  2104. metadata = self.get_metadata(self.PKG_INFO)
  2105. self._pkg_info = Parser().parsestr(metadata)
  2106. return self._pkg_info
  2107. @property
  2108. def _dep_map(self):
  2109. try:
  2110. return self.__dep_map
  2111. except AttributeError:
  2112. self.__dep_map = self._compute_dependencies()
  2113. return self.__dep_map
  2114. def _preparse_requirement(self, requires_dist):
  2115. """Convert 'Foobar (1); baz' to ('Foobar ==1', 'baz')
  2116. Split environment marker, add == prefix to version specifiers as
  2117. necessary, and remove parenthesis.
  2118. """
  2119. parts = requires_dist.split(';', 1) + ['']
  2120. distvers = parts[0].strip()
  2121. mark = parts[1].strip()
  2122. distvers = re.sub(self.EQEQ, r"\1==\2\3", distvers)
  2123. distvers = distvers.replace('(', '').replace(')', '')
  2124. return (distvers, mark)
  2125. def _compute_dependencies(self):
  2126. """Recompute this distribution's dependencies."""
  2127. from _markerlib import compile as compile_marker
  2128. dm = self.__dep_map = {None: []}
  2129. reqs = []
  2130. # Including any condition expressions
  2131. for req in self._parsed_pkg_info.get_all('Requires-Dist') or []:
  2132. distvers, mark = self._preparse_requirement(req)
  2133. parsed = next(parse_requirements(distvers))
  2134. parsed.marker_fn = compile_marker(mark)
  2135. reqs.append(parsed)
  2136. def reqs_for_extra(extra):
  2137. for req in reqs:
  2138. if req.marker_fn(override={'extra':extra}):
  2139. yield req
  2140. common = frozenset(reqs_for_extra(None))
  2141. dm[None].extend(common)
  2142. for extra in self._parsed_pkg_info.get_all('Provides-Extra') or []:
  2143. extra = safe_extra(extra.strip())
  2144. dm[extra] = list(frozenset(reqs_for_extra(extra)) - common)
  2145. return dm
  2146. _distributionImpl = {
  2147. '.egg': Distribution,
  2148. '.egg-info': Distribution,
  2149. '.dist-info': DistInfoDistribution,
  2150. }
  2151. def issue_warning(*args,**kw):
  2152. level = 1
  2153. g = globals()
  2154. try:
  2155. # find the first stack frame that is *not* code in
  2156. # the pkg_resources module, to use for the warning
  2157. while sys._getframe(level).f_globals is g:
  2158. level += 1
  2159. except ValueError:
  2160. pass
  2161. from warnings import warn
  2162. warn(stacklevel = level+1, *args, **kw)
  2163. def parse_requirements(strs):
  2164. """Yield ``Requirement`` objects for each specification in `strs`
  2165. `strs` must be an instance of ``basestring``, or a (possibly-nested)
  2166. iterable thereof.
  2167. """
  2168. # create a steppable iterator, so we can handle \-continuations
  2169. lines = iter(yield_lines(strs))
  2170. def scan_list(ITEM, TERMINATOR, line, p, groups, item_name):
  2171. items = []
  2172. while not TERMINATOR(line, p):
  2173. if CONTINUE(line, p):
  2174. try:
  2175. line = next(lines)
  2176. p = 0
  2177. except StopIteration:
  2178. raise ValueError(
  2179. "\\ must not appear on the last nonblank line"
  2180. )
  2181. match = ITEM(line, p)
  2182. if not match:
  2183. msg = "Expected " + item_name + " in"
  2184. raise ValueError(msg, line, "at", line[p:])
  2185. items.append(match.group(*groups))
  2186. p = match.end()
  2187. match = COMMA(line, p)
  2188. if match:
  2189. # skip the comma
  2190. p = match.end()
  2191. elif not TERMINATOR(line, p):
  2192. msg = "Expected ',' or end-of-list in"
  2193. raise ValueError(msg, line, "at", line[p:])
  2194. match = TERMINATOR(line, p)
  2195. if match: p = match.end() # skip the terminator, if any
  2196. return line, p, items
  2197. for line in lines:
  2198. match = DISTRO(line)
  2199. if not match:
  2200. raise ValueError("Missing distribution spec", line)
  2201. project_name = match.group(1)
  2202. p = match.end()
  2203. extras = []
  2204. match = OBRACKET(line, p)
  2205. if match:
  2206. p = match.end()
  2207. line, p, extras = scan_list(
  2208. DISTRO, CBRACKET, line, p, (1,), "'extra' name"
  2209. )
  2210. line, p, specs = scan_list(VERSION, LINE_END, line, p, (1, 2),
  2211. "version spec")
  2212. specs = [(op, safe_version(val)) for op, val in specs]
  2213. yield Requirement(project_name, specs, extras)
  2214. class Requirement:
  2215. def __init__(self, project_name, specs, extras):
  2216. """DO NOT CALL THIS UNDOCUMENTED METHOD; use Requirement.parse()!"""
  2217. self.unsafe_name, project_name = project_name, safe_name(project_name)
  2218. self.project_name, self.key = project_name, project_name.lower()
  2219. index = [
  2220. (parse_version(v), state_machine[op], op, v)
  2221. for op, v in specs
  2222. ]
  2223. index.sort()
  2224. self.specs = [(op, ver) for parsed, trans, op, ver in index]
  2225. self.index, self.extras = index, tuple(map(safe_extra, extras))
  2226. self.hashCmp = (
  2227. self.key,
  2228. tuple((op, parsed) for parsed, trans, op, ver in index),
  2229. frozenset(self.extras),
  2230. )
  2231. self.__hash = hash(self.hashCmp)
  2232. def __str__(self):
  2233. specs = ','.join([''.join(s) for s in self.specs])
  2234. extras = ','.join(self.extras)
  2235. if extras: extras = '[%s]' % extras
  2236. return '%s%s%s' % (self.project_name, extras, specs)
  2237. def __eq__(self, other):
  2238. return isinstance(other, Requirement) and self.hashCmp==other.hashCmp
  2239. def __contains__(self, item):
  2240. if isinstance(item, Distribution):
  2241. if item.key != self.key:
  2242. return False
  2243. # only get if we need it
  2244. if self.index:
  2245. item = item.parsed_version
  2246. elif isinstance(item, basestring):
  2247. item = parse_version(item)
  2248. last = None
  2249. # -1, 0, 1
  2250. compare = lambda a, b: (a > b) - (a < b)
  2251. for parsed, trans, op, ver in self.index:
  2252. # Indexing: 0, 1, -1
  2253. action = trans[compare(item, parsed)]
  2254. if action=='F':
  2255. return False
  2256. elif action=='T':
  2257. return True
  2258. elif action=='+':
  2259. last = True
  2260. elif action=='-' or last is None:
  2261. last = False
  2262. # no rules encountered
  2263. if last is None: last = True
  2264. return last
  2265. def __hash__(self):
  2266. return self.__hash
  2267. def __repr__(self): return "Requirement.parse(%r)" % str(self)
  2268. @staticmethod
  2269. def parse(s):
  2270. reqs = list(parse_requirements(s))
  2271. if reqs:
  2272. if len(reqs)==1:
  2273. return reqs[0]
  2274. raise ValueError("Expected only one requirement", s)
  2275. raise ValueError("No requirements found", s)
  2276. state_machine = {
  2277. # =><
  2278. '<': '--T',
  2279. '<=': 'T-T',
  2280. '>': 'F+F',
  2281. '>=': 'T+F',
  2282. '==': 'T..',
  2283. '!=': 'F++',
  2284. }
  2285. def _get_mro(cls):
  2286. """Get an mro for a type or classic class"""
  2287. if not isinstance(cls, type):
  2288. class cls(cls, object): pass
  2289. return cls.__mro__[1:]
  2290. return cls.__mro__
  2291. def _find_adapter(registry, ob):
  2292. """Return an adapter factory for `ob` from `registry`"""
  2293. for t in _get_mro(getattr(ob, '__class__', type(ob))):
  2294. if t in registry:
  2295. return registry[t]
  2296. def ensure_directory(path):
  2297. """Ensure that the parent directory of `path` exists"""
  2298. dirname = os.path.dirname(path)
  2299. if not os.path.isdir(dirname):
  2300. os.makedirs(dirname)
  2301. def split_sections(s):
  2302. """Split a string or iterable thereof into (section, content) pairs
  2303. Each ``section`` is a stripped version of the section header ("[section]")
  2304. and each ``content`` is a list of stripped lines excluding blank lines and
  2305. comment-only lines. If there are any such lines before the first section
  2306. header, they're returned in a first ``section`` of ``None``.
  2307. """
  2308. section = None
  2309. content = []
  2310. for line in yield_lines(s):
  2311. if line.startswith("["):
  2312. if line.endswith("]"):
  2313. if section or content:
  2314. yield section, content
  2315. section = line[1:-1].strip()
  2316. content = []
  2317. else:
  2318. raise ValueError("Invalid section heading", line)
  2319. else:
  2320. content.append(line)
  2321. # wrap up last segment
  2322. yield section, content
  2323. def _mkstemp(*args,**kw):
  2324. from tempfile import mkstemp
  2325. old_open = os.open
  2326. try:
  2327. # temporarily bypass sandboxing
  2328. os.open = os_open
  2329. return mkstemp(*args,**kw)
  2330. finally:
  2331. # and then put it back
  2332. os.open = old_open
  2333. # Set up global resource manager (deliberately not state-saved)
  2334. _manager = ResourceManager()
  2335. def _initialize(g):
  2336. for name in dir(_manager):
  2337. if not name.startswith('_'):
  2338. g[name] = getattr(_manager, name)
  2339. _initialize(globals())
  2340. # Prepare the master working set and make the ``require()`` API available
  2341. working_set = WorkingSet._build_master()
  2342. _declare_state('object', working_set=working_set)
  2343. require = working_set.require
  2344. iter_entry_points = working_set.iter_entry_points
  2345. add_activation_listener = working_set.subscribe
  2346. run_script = working_set.run_script
  2347. # backward compatibility
  2348. run_main = run_script
  2349. # Activate all distributions already on sys.path, and ensure that
  2350. # all distributions added to the working set in the future (e.g. by
  2351. # calling ``require()``) will get activated as well.
  2352. add_activation_listener(lambda dist: dist.activate())
  2353. working_set.entries=[]
  2354. list(map(working_set.add_entry, sys.path)) # match order