zeroinstall.zerostore

1 """ 2 Code for managing the implementation cache. 3 """ 4 5 # Copyright (C) 2009, Thomas Leonard 6 # See the README file for details, or visit http://0install.net. 7 8 from zeroinstall import _ 9 import os 10 from logging import debug, info, warn 11 12 from zeroinstall.support import basedir 13 from zeroinstall import SafeException, support 14

15 -class BadDigest(SafeException):

16 """Thrown if a digest is invalid (either syntactically or cryptographically).""" 17 detail = None

18

19 -class NotStored(SafeException):

20 """Throws if a requested implementation isn't in the cache."""

21

22 -class NonwritableStore(SafeException):

23 """Attempt to add to a non-writable store directory."""

24

25 -def _copytree2(src, dst):

26 import shutil 27 names = os.listdir(src) 28 assert os.path.isdir(dst) 29 for name in names: 30 srcname = os.path.join(src, name) 31 dstname = os.path.join(dst, name) 32 if os.path.islink(srcname): 33 linkto = os.readlink(srcname) 34 os.symlink(linkto, dstname) 35 elif os.path.isdir(srcname): 36 os.mkdir(dstname) 37 mtime = int(os.lstat(srcname).st_mtime) 38 _copytree2(srcname, dstname) 39 os.utime(dstname, (mtime, mtime)) 40 else: 41 shutil.copy2(srcname, dstname)

42

43 -class Store:

44 """A directory for storing implementations.""" 45

46 - def __init__(self, dir, public = False):

47 """Create a new Store. 48 @param dir: directory to contain the implementations 49 @type dir: str 50 @param public: deprecated 51 @type public: bool""" 52 self.dir = dir

53

54 - def __str__(self):

55 return _("Store '%s'") % self.dir

56

57 - def lookup(self, digest):

58 try: 59 alg, value = digest.split('=', 1) 60 except ValueError: 61 raise BadDigest(_("Digest must be in the form ALG=VALUE, not '%s'") % digest) 62 try: 63 assert '/' not in value 64 assert value not in ('', '.', '..') 65 except ValueError as ex: 66 raise BadDigest(_("Bad value for digest: %s") % str(ex)) 67 dir = os.path.join(self.dir, digest) 68 if os.path.isdir(dir): 69 return dir 70 return None

71

72 - def get_tmp_dir_for(self, required_digest):

73 """Create a temporary directory in the directory where we would store an implementation 74 with the given digest. This is used to setup a new implementation before being renamed if 75 it turns out OK. 76 @raise NonwritableStore: if we can't create it""" 77 try: 78 if not os.path.isdir(self.dir): 79 os.makedirs(self.dir) 80 from tempfile import mkdtemp 81 tmp = mkdtemp(dir = self.dir, prefix = 'tmp-') 82 os.chmod(tmp, 0o755) # r-x for all; needed by 0store-helper 83 return tmp 84 except OSError as ex: 85 raise NonwritableStore(str(ex))

86

87 - def add_archive_to_cache(self, required_digest, data, url, extract = None, type = None, start_offset = 0, try_helper = False):

88 from . import unpack 89 90 if self.lookup(required_digest): 91 info(_("Not adding %s as it already exists!"), required_digest) 92 return 93 94 tmp = self.get_tmp_dir_for(required_digest) 95 try: 96 unpack.unpack_archive(url, data, tmp, extract, type = type, start_offset = start_offset) 97 except: 98 import shutil 99 shutil.rmtree(tmp) 100 raise 101 102 try: 103 self.check_manifest_and_rename(required_digest, tmp, extract, try_helper = try_helper) 104 except Exception: 105 #warn(_("Leaving extracted directory as %s"), tmp) 106 support.ro_rmtree(tmp) 107 raise

108

109 - def add_dir_to_cache(self, required_digest, path, try_helper = False):

110 """Copy the contents of path to the cache. 111 @param required_digest: the expected digest 112 @type required_digest: str 113 @param path: the root of the tree to copy 114 @type path: str 115 @param try_helper: attempt to use privileged helper before user cache (since 0.26) 116 @type try_helper: bool 117 @raise BadDigest: if the contents don't match the given digest.""" 118 if self.lookup(required_digest): 119 info(_("Not adding %s as it already exists!"), required_digest) 120 return 121 122 tmp = self.get_tmp_dir_for(required_digest) 123 try: 124 _copytree2(path, tmp) 125 self.check_manifest_and_rename(required_digest, tmp, try_helper = try_helper) 126 except: 127 warn(_("Error importing directory.")) 128 warn(_("Deleting %s"), tmp) 129 support.ro_rmtree(tmp) 130 raise

131

132 - def _add_with_helper(self, required_digest, path):

133 """Use 0store-secure-add to copy 'path' to the system store. 134 @param required_digest: the digest for path 135 @type required_digest: str 136 @param path: root of implementation directory structure 137 @type path: str 138 @return: True iff the directory was copied into the system cache successfully 139 """ 140 if required_digest.startswith('sha1='): 141 return False # Old digest alg not supported 142 helper = support.find_in_path('0store-secure-add-helper') 143 if not helper: 144 info(_("'0store-secure-add-helper' command not found. Not adding to system cache.")) 145 return False 146 import subprocess 147 env = os.environ.copy() 148 env['ENV_NOT_CLEARED'] = 'Unclean' # (warn about insecure configurations) 149 env['HOME'] = 'Unclean' # (warn about insecure configurations) 150 dev_null = os.open('/dev/null', os.O_RDONLY) 151 try: 152 info(_("Trying to add to system cache using %s"), helper) 153 child = subprocess.Popen([helper, required_digest], 154 stdin = dev_null, 155 cwd = path, 156 env = env) 157 exit_code = child.wait() 158 finally: 159 os.close(dev_null) 160 161 if exit_code: 162 warn(_("0store-secure-add-helper failed.")) 163 return False 164 165 info(_("Added succcessfully.")) 166 return True

167

168 - def check_manifest_and_rename(self, required_digest, tmp, extract = None, try_helper = False):

169 """Check that tmp[/extract] has the required_digest. 170 On success, rename the checked directory to the digest, and 171 make the whole tree read-only. 172 @param try_helper: attempt to use privileged helper to import to system cache first (since 0.26) 173 @type try_helper: bool 174 @raise BadDigest: if the input directory doesn't match the given digest""" 175 if extract: 176 extracted = os.path.join(tmp, extract) 177 if not os.path.isdir(extracted): 178 raise Exception(_('Directory %s not found in archive') % extract) 179 else: 180 extracted = tmp 181 182 from . import manifest 183 184 manifest.fixup_permissions(extracted) 185 186 alg, required_value = manifest.splitID(required_digest) 187 actual_digest = alg.getID(manifest.add_manifest_file(extracted, alg)) 188 if actual_digest != required_digest: 189 raise BadDigest(_('Incorrect manifest -- archive is corrupted.\n' 190 'Required digest: %(required_digest)s\n' 191 'Actual digest: %(actual_digest)s\n') % 192 {'required_digest': required_digest, 'actual_digest': actual_digest}) 193 194 if try_helper: 195 if self._add_with_helper(required_digest, extracted): 196 support.ro_rmtree(tmp) 197 return 198 info(_("Can't add to system store. Trying user store instead.")) 199 200 info(_("Caching new implementation (digest %s) in %s"), required_digest, self.dir) 201 202 final_name = os.path.join(self.dir, required_digest) 203 if os.path.isdir(final_name): 204 raise Exception(_("Item %s already stored.") % final_name) # XXX: not really an error 205 206 # If we just want a subdirectory then the rename will change 207 # extracted/.. and so we'll need write permission on 'extracted' 208 209 os.chmod(extracted, 0o755) 210 os.rename(extracted, final_name) 211 os.chmod(final_name, 0o555) 212 213 if extract: 214 os.rmdir(tmp)

215

216 - def __repr__(self):

217 return "<store: %s>" % self.dir

218

219 -class Stores(object):

220 """A list of L{Store}s. All stores are searched when looking for an implementation. 221 When storing, we use the first of the system caches (if writable), or the user's 222 cache otherwise.""" 223 __slots__ = ['stores'] 224

225 - def __init__(self):

226 user_store = os.path.join(basedir.xdg_cache_home, '0install.net', 'implementations') 227 self.stores = [Store(user_store)] 228 229 impl_dirs = basedir.load_first_config('0install.net', 'injector', 230 'implementation-dirs') 231 debug(_("Location of 'implementation-dirs' config file being used: '%s'"), impl_dirs) 232 if impl_dirs: 233 dirs = open(impl_dirs) 234 else: 235 if os.name == "nt": 236 from win32com.shell import shell, shellcon 237 localAppData = shell.SHGetFolderPath(0, shellcon.CSIDL_LOCAL_APPDATA, 0, 0) 238 commonAppData = shell.SHGetFolderPath(0, shellcon.CSIDL_COMMON_APPDATA, 0, 0) 239 240 userCache = os.path.join(localAppData, "0install.net", "implementations") 241 sharedCache = os.path.join(commonAppData, "0install.net", "implementations") 242 dirs = [userCache, sharedCache] 243 244 else: 245 dirs = ['/var/cache/0install.net/implementations'] 246 247 for directory in dirs: 248 directory = directory.strip() 249 if directory and not directory.startswith('#'): 250 debug(_("Added system store '%s'"), directory) 251 self.stores.append(Store(directory))

252

253 - def lookup(self, digest):

254 """@deprecated: use lookup_any instead""" 255 return self.lookup_any([digest])

256

257 - def lookup_any(self, digests):

258 """Search for digest in all stores. 259 @raises NotStored: if not found""" 260 path = self.lookup_maybe(digests) 261 if path: 262 return path 263 raise NotStored(_("Item with digests '%(digests)s' not found in stores. Searched:\n- %(stores)s") % 264 {'digests': digests, 'stores': '\n- '.join([s.dir for s in self.stores])})

265

266 - def lookup_maybe(self, digests):

267 """Like lookup_any, but return None if it isn't found. 268 @since: 0.53""" 269 assert digests 270 for digest in digests: 271 assert digest 272 if '/' in digest or '=' not in digest: 273 raise BadDigest(_('Syntax error in digest (use ALG=VALUE, not %s)') % digest) 274 for store in self.stores: 275 path = store.lookup(digest) 276 if path: 277 return path 278 return None

279

280 - def add_dir_to_cache(self, required_digest, dir):

281 """Add to the best writable cache. 282 @see: L{Store.add_dir_to_cache}""" 283 self._write_store(lambda store, **kwargs: store.add_dir_to_cache(required_digest, dir, **kwargs))

284

285 - def add_archive_to_cache(self, required_digest, data, url, extract = None, type = None, start_offset = 0):

286 """Add to the best writable cache. 287 @see: L{Store.add_archive_to_cache}""" 288 self._write_store(lambda store, **kwargs: store.add_archive_to_cache(required_digest, 289 data, url, extract, type = type, start_offset = start_offset, **kwargs))

290

291 - def _write_store(self, fn):

292 """Call fn(first_system_store). If it's read-only, try again with the user store.""" 293 if len(self.stores) > 1: 294 try: 295 fn(self.get_first_system_store()) 296 return 297 except NonwritableStore: 298 debug(_("%s not-writable. Trying helper instead."), self.get_first_system_store()) 299 pass 300 fn(self.stores[0], try_helper = True)

301

302 - def get_first_system_store(self):

303 """The first system store is the one we try writing to first. 304 @since: 0.30""" 305 try: 306 return self.stores[1] 307 except IndexError: 308 raise SafeException(_("No system stores have been configured"))

309

Source Code for Package zeroinstall.zerostore