Package entropy :: Package server :: Package interfaces :: Module main

Source Code for Module entropy.server.interfaces.main

   1  # -*- coding: utf-8 -*- 
   2  """ 
   3   
   4      @author: Fabio Erculiani <[email protected]> 
   5      @author: Slawomir Nizio <[email protected]> 
   6      @contact: [email protected] 
   7      @copyright: Fabio Erculiani, Slawomir Nizio 
   8      @license: GPL-2 
   9   
  10      B{Entropy Package Manager Server Main Interfaces}. 
  11   
  12  """ 
  13  import codecs 
  14  import collections 
  15  import copy 
  16  import errno 
  17  import hashlib 
  18  import os 
  19  import re 
  20  import shutil 
  21  import stat 
  22  import subprocess 
  23  import sys 
  24  import time 
  25  import threading 
  26   
  27  from entropy.exceptions import OnlineMirrorError, PermissionDenied, \ 
  28      SystemDatabaseError, RepositoryError 
  29  from entropy.const import etpConst, etpSys, const_setup_perms, \ 
  30      const_create_working_dirs, const_convert_to_unicode, \ 
  31      const_setup_file, const_get_stringtype, const_debug_write, \ 
  32      const_debug_enabled, const_convert_to_rawstring, const_mkdtemp, \ 
  33      const_mkstemp, const_file_readable 
  34  from entropy.output import purple, red, darkgreen, \ 
  35      bold, brown, blue, darkred, teal 
  36  from entropy.cache import EntropyCacher 
  37  from entropy.server.interfaces.mirrors import Server as MirrorsServer 
  38  from entropy.i18n import _ 
  39  from entropy.core import BaseConfigParser 
  40  from entropy.core.settings.base import SystemSettings 
  41  from entropy.core.settings.plugins.skel import SystemSettingsPlugin 
  42  from entropy.transceivers import EntropyTransceiver 
  43  from entropy.db import EntropyRepository 
  44  from entropy.db.skel import EntropyRepositoryPlugin 
  45  from entropy.server.interfaces.db import ServerRepositoryStatus, \ 
  46      ServerPackagesRepository 
  47  from entropy.spm.plugins.factory import get_default_instance as get_spm, \ 
  48      get_default_class as get_spm_class 
  49  from entropy.qa import QAInterfacePlugin 
  50  from entropy.security import Repository as RepositorySecurity 
  51  from entropy.db.exceptions import ProgrammingError 
  52  from entropy.client.interfaces import Client 
  53  from entropy.client.interfaces.db import InstalledPackagesRepository, \ 
  54      GenericRepository 
  55  from entropy.client.misc import ConfigurationUpdates, ConfigurationFiles 
  56   
  57  import entropy.dep 
  58  import entropy.tools 
  59  import entropy.dump 
  60   
  61  SERVER_QA_PLUGIN = "ServerQAInterfacePlugin" 
62 63 64 -class ServerEntropyRepositoryPlugin(EntropyRepositoryPlugin):
65 66 PLUGIN_ID = "__server__" 67
68 - def __init__(self, server_interface, metadata = None):
69 """ 70 Entropy server-side repository ServerPackagesRepository Plugin class. 71 This class will be instantiated and automatically added to 72 ServerPackagesRepository instances generated by Entropy Server. 73 74 @param server_interface: Entropy Server interface instance 75 @type server_interface: entropy.server.interfaces.Server class 76 @param metadata: any dict form metadata map (key => value) 77 @type metadata: dict 78 """ 79 EntropyRepositoryPlugin.__init__(self) 80 self._cacher = EntropyCacher() 81 self._settings = SystemSettings() 82 self.srv_sys_settings_plugin = \ 83 etpConst['system_settings_plugins_ids']['server_plugin'] 84 self._server = server_interface 85 if metadata is None: 86 self._metadata = {} 87 else: 88 self._metadata = metadata
89
90 - def get_id(self):
92
93 - def get_metadata(self):
94 """ 95 This method should always return a direct reference to the object and 96 NOT a copy. 97 """ 98 return self._metadata
99
100 - def add_plugin_hook(self, entropy_repository_instance):
101 const_debug_write(__name__, 102 "ServerEntropyRepositoryPlugin: calling add_plugin_hook => %s" % ( 103 self,) 104 ) 105 106 repo = entropy_repository_instance.repository_id() 107 local_dbfile = self._metadata['local_dbfile'] 108 if local_dbfile is not None: 109 taint_file = self._server._get_local_repository_taint_file( 110 repo) 111 if os.path.isfile(taint_file): 112 dbs = ServerRepositoryStatus() 113 dbs.set_tainted(local_dbfile) 114 dbs.set_bumped(local_dbfile) 115 116 if "__temporary__" in self._metadata: # in-memory db? 117 local_dbfile_exists = True 118 else: 119 local_dbfile_exists = os.path.lexists(local_dbfile) 120 121 if not local_dbfile_exists: 122 # better than having a completely broken db 123 self._metadata['read_only'] = False 124 # force parameters, only ServerEntropyRepository exposes 125 # the setReadonly method 126 entropy_repository_instance.setReadonly(False) 127 entropy_repository_instance.initializeRepository() 128 entropy_repository_instance.commit() 129 130 out_intf = self._metadata.get('output_interface') 131 if out_intf is not None: 132 entropy_repository_instance.output = out_intf.output 133 entropy_repository_instance.ask_question = out_intf.ask_question 134 135 return 0
136
137 - def close_repo_hook(self, entropy_repository_instance):
138 const_debug_write(__name__, 139 "ServerEntropyRepositoryPlugin: calling close_repo_hook => %s" % ( 140 self,) 141 ) 142 143 # this happens because close_repositories() might be called 144 # before _setup_services() and in general, at any time, so, in this 145 # case, there is no need to print bullshit to dev. 146 if self._server.Mirrors is None: 147 return 0 148 149 repo = entropy_repository_instance.repository_id() 150 dbfile = self._metadata['local_dbfile'] 151 if dbfile is None: 152 # fake repo, or temporary one 153 return 0 154 155 read_only = self._metadata['read_only'] 156 if not read_only: 157 sts = ServerRepositoryStatus() 158 if sts.is_tainted(dbfile) and not sts.is_unlock_msg(dbfile): 159 u_msg = "[%s] %s" % (brown(repo), 160 darkgreen(_("mirrors have not been unlocked. Sync them.")),) 161 self._server.output( 162 u_msg, 163 importance = 1, 164 level = "warning", 165 header = brown(" * ") 166 ) 167 # avoid spamming 168 sts.set_unlock_msg(dbfile) 169 170 return 0
171
172 - def commit_hook(self, entropy_repository_instance):
173 174 const_debug_write(__name__, 175 "ServerEntropyRepositoryPlugin: calling commit_hook => %s" % ( 176 self,) 177 ) 178 179 dbs = ServerRepositoryStatus() 180 dbfile = self._metadata['local_dbfile'] 181 if dbfile is None: 182 # fake repo, or temporary one 183 return 0 184 repo = entropy_repository_instance.repository_id() 185 read_only = self._metadata['read_only'] 186 if read_only: 187 # do not taint database 188 return 0 189 190 # taint the database status 191 taint_file = self._server._get_local_repository_taint_file(repo) 192 enc = etpConst['conf_encoding'] 193 with codecs.open(taint_file, "w", encoding=enc) as f: 194 f.write("repository tainted\n") 195 196 const_setup_file(taint_file, etpConst['entropygid'], 0o664) 197 dbs.set_tainted(dbfile) 198 199 if not dbs.is_bumped(dbfile): 200 # bump revision, setting DatabaseBump causes 201 # the session to just bump once 202 dbs.set_bumped(dbfile) 203 """ 204 Entropy repository revision bumping function. 205 Every time it's called, 206 revision is incremented by 1. 207 """ 208 revision_file = self._server._get_local_repository_revision_file( 209 repo) 210 enc = etpConst['conf_encoding'] 211 if not os.path.isfile(revision_file): 212 revision = 1 213 else: 214 with codecs.open(revision_file, "r", encoding=enc) as rev_f: 215 revision = int(rev_f.readline().strip()) 216 revision += 1 217 218 tmp_revision_file = revision_file + ".tmp" 219 with codecs.open(tmp_revision_file, "w", encoding=enc) as rev_fw: 220 rev_fw.write(str(revision)+"\n") 221 222 # atomic ! 223 os.rename(tmp_revision_file, revision_file) 224 225 if not dbs.are_sets_synced(dbfile): 226 # auto-update package sets 227 self._server._sync_package_sets(entropy_repository_instance) 228 dbs.set_synced_sets(dbfile) 229 230 return 0
231
232 - def _get_category_description_from_disk(self, category):
233 """ 234 Get category name description from Source Package Manager. 235 236 @param category: category name 237 @type category: string 238 @return: category description 239 @rtype: string 240 """ 241 spm = self._server.Spm() 242 return spm.get_package_category_description_metadata(category)
243
244 - def __save_rss(self, srv_repo, rss_name, srv_updates):
245 # save to disk 246 try: 247 self._cacher.save(rss_name, srv_updates, 248 cache_dir = Server.CACHE_DIR) 249 except IOError as err: 250 e_msg = "[%s] %s: %s" % (brown(srv_repo), 251 purple(_("cannot store updates RSS cache")), 252 repr(err),) 253 self._server.output( 254 e_msg, 255 importance = 1, 256 level = "warning", 257 header = brown(" * ") 258 )
259
260 - def _write_rss_for_removed_package(self, repo_db, package_id):
261 262 # setup variables we're going to use 263 srv_repo = repo_db.repository_id() 264 rss_revision = repo_db.retrieveRevision(package_id) 265 rss_atom = "%s~%s" % (repo_db.retrieveAtom(package_id), rss_revision,) 266 status = ServerRepositoryStatus() 267 srv_updates = status.get_updates_log(srv_repo) 268 rss_name = srv_repo + etpConst['rss-dump-name'] 269 270 # load metadata from on disk cache, if available 271 rss_obj = self._cacher.pop(rss_name, cache_dir = Server.CACHE_DIR) 272 if rss_obj: 273 srv_updates.update(rss_obj) 274 275 # setup metadata keys, if not available 276 if 'added' not in srv_updates: 277 srv_updates['added'] = {} 278 if 'removed' not in srv_updates: 279 srv_updates['removed'] = {} 280 if 'light' not in srv_updates: 281 srv_updates['light'] = {} 282 283 # if pkgatom (rss_atom) is in the "added" metadata, drop it 284 if rss_atom in srv_updates['added']: 285 del srv_updates['added'][rss_atom] 286 # same thing for light key 287 if rss_atom in srv_updates['light']: 288 del srv_updates['light'][rss_atom] 289 290 # add metadata 291 mydict = {} 292 try: 293 mydict['description'] = repo_db.retrieveDescription(package_id) 294 except TypeError: 295 mydict['description'] = "N/A" 296 try: 297 mydict['homepage'] = repo_db.retrieveHomepage(package_id) 298 except TypeError: 299 mydict['homepage'] = "" 300 srv_updates['removed'][rss_atom] = mydict 301 302 # save to disk 303 self.__save_rss(srv_repo, rss_name, srv_updates)
304
305 - def _write_rss_for_added_package(self, repo_db, package_id, package_data):
306 307 # setup variables we're going to use 308 srv_repo = repo_db.repository_id() 309 rss_atom = "%s~%s" % (package_data['atom'], package_data['revision'],) 310 status = ServerRepositoryStatus() 311 srv_updates = status.get_updates_log(srv_repo) 312 rss_name = srv_repo + etpConst['rss-dump-name'] 313 314 # load metadata from on disk cache, if available 315 rss_obj = self._cacher.pop(rss_name, cache_dir = Server.CACHE_DIR) 316 if rss_obj: 317 srv_updates.update(rss_obj) 318 319 # setup metadata keys, if not available 320 if 'added' not in srv_updates: 321 srv_updates['added'] = {} 322 if 'removed' not in srv_updates: 323 srv_updates['removed'] = {} 324 if 'light' not in srv_updates: 325 srv_updates['light'] = {} 326 327 # if package_data['atom'] (rss_atom) is in the 328 # "removed" metadata, drop it 329 if rss_atom in srv_updates['removed']: 330 del srv_updates['removed'][rss_atom] 331 332 # add metadata 333 srv_updates['added'][rss_atom] = {} 334 srv_updates['added'][rss_atom]['description'] = \ 335 package_data['description'] 336 srv_updates['added'][rss_atom]['homepage'] = \ 337 package_data['homepage'] 338 339 srv_updates['light'][rss_atom] = {} 340 srv_updates['light'][rss_atom]['description'] = \ 341 package_data['description'] 342 srv_updates['light'][rss_atom]['homepage'] = \ 343 package_data['homepage'] 344 srv_updates['light'][rss_atom]['package_id'] = package_id 345 date_raw_str = const_convert_to_rawstring(package_data['datecreation']) 346 srv_updates['light'][rss_atom]['time_hash'] = \ 347 hashlib.sha256(date_raw_str).hexdigest() 348 349 # save to disk 350 self.__save_rss(srv_repo, rss_name, srv_updates)
351
352 - def add_package_hook(self, entropy_repository_instance, package_id, 353 package_data):
354 355 const_debug_write(__name__, 356 "ServerEntropyRepositoryPlugin: calling add_package_hook => %s" % ( 357 self,) 358 ) 359 360 # handle server-side repo RSS support 361 sys_set_plug = self.srv_sys_settings_plugin 362 if self._settings[sys_set_plug]['server']['rss']['enabled']: 363 self._write_rss_for_added_package(entropy_repository_instance, 364 package_id, package_data) 365 366 try: 367 descdata = self._get_category_description_from_disk( 368 package_data['category']) 369 entropy_repository_instance.setCategoryDescription( 370 package_data['category'], descdata) 371 except (IOError, OSError, EOFError,): 372 pass 373 entropy_repository_instance.commit() 374 375 return 0
376
377 - def remove_package_hook(self, entropy_repository_instance, package_id, 378 from_add_package):
379 380 const_debug_write(__name__, 381 "ServerEntropyRepositoryPlugin: calling remove_package_hook => %s" % ( 382 self,) 383 ) 384 385 # handle server-side repo RSS support 386 sys_set_plug = self.srv_sys_settings_plugin 387 if self._settings[sys_set_plug]['server']['rss']['enabled'] \ 388 and (not from_add_package): 389 390 # store addPackage action 391 self._write_rss_for_removed_package(entropy_repository_instance, 392 package_id) 393 394 return 0
395
396 - def treeupdates_move_action_hook(self, entropy_repository_instance, 397 package_id):
398 # check for injection and warn the developer 399 injected = entropy_repository_instance.isInjected(package_id) 400 new_atom = entropy_repository_instance.retrieveAtom(package_id) 401 if injected: 402 mytxt = "%s: %s %s. %s !!! %s." % ( 403 bold(_("INJECT")), 404 blue(str(new_atom)), 405 red(_("has been injected")), 406 red(_("quickpkg manually to update embedded db")), 407 red(_("Repository updated anyway")), 408 ) 409 self._server.output( 410 mytxt, 411 importance = 1, 412 level = "warning", 413 header = darkred(" * ") 414 ) 415 return 0
416
417 - def treeupdates_slot_move_action_hook(self, entropy_repository_instance, 418 package_id):
419 return self.treeupdates_move_action_hook(entropy_repository_instance, 420 package_id)
421
422 423 -class RepositoryConfigParser(BaseConfigParser):
424 """ 425 Entropy .ini-like server-side repository configuration file parser. 426 427 Entropy Server now supports repositories defined inside 428 /etc/entropy/repositories.conf.d/ files, written using the 429 syntax detailed below. This improves the ability to enable, disable, 430 add and remove repositories programmatically. Furthermore, it 431 makes possible to extend the supported parameters without breaking 432 backward compatibility. 433 434 In order to differentiate Entropy Client repository definitions between 435 Entropy Server ones, each repository section must start with "[server=". 436 437 This is an example of the syntax (with a complete listing 438 of the supported arguments): 439 440 [server=sabayon-limbo] 441 desc = Sabayon Linux Official Testing Repository 442 repo = ssh://[email protected]:~username/sabayon-limbo 443 enabled = <true/false> 444 445 [server=sabayon-limbo] 446 desc = This statement will be ignored. 447 repo-only = ssh://[email protected]:~username/sabayon-limbo 448 pkg-only = ssh://[email protected]:~username/sabayon-limbo 449 450 [server=sabayon-base] 451 desc = This is the base repository. 452 repo-only = ssh://[email protected]:~username/sabayon-base 453 pkg-only = ssh://[email protected]:~username/sabayon-base 454 base = <true/false> 455 456 As you can see, multiple statements for the same repository 457 are allowed. However, only the first desc = statement will be 458 considered, while there can be as many {pkg,repo}* = as you want. 459 460 The repository order is important, but this is guaranteed by the 461 fact that configuration files are parsed in lexical order. 462 463 Statements description: 464 - "desc": stands for description, the repository name description. 465 - "repo": the push & pull URI, for both packages and repository database. 466 - "repo-only": same as repo, but only for the repository database 467 push & pull. 468 - "pkg-only": same as repo, but only for the packages push & pull. 469 The supported protocols are those supported by entropy.fetchers. 470 - "enabled": if set, its value can be either "true" or "false". The default 471 value is "true". It indicates if a repository is configured 472 but currently disabled or enabled. Please take into account 473 that config files in /etc/entropy/repositories.conf.d/ starting 474 with "_" are considered to contain disabled repositories. This 475 is just provided for convienence. 476 - "base": if set, its value can be either "true" or "false". The default 477 value is "false". If no repository has the flag set, the first 478 listed repository will be the base one. Only the first repository 479 with "base = true" will be considered. The base repository is the 480 repository that is considered base for all the others 481 (the main one). 482 - "exclude-qa": if set, its value can be either "true" or "false". 483 The default value is "false". If "true", the repository is 484 excluded from QA checks. 485 """ 486 487 _SUPPORTED_KEYS = ("desc", "repo", "repo-only", "pkg-only", 488 "base", "enabled", "exclude-qa") 489 490 _DEFAULT_ENABLED_VALUE = True 491 _DEFAULT_QA_VALUE = False 492 _DEFAULT_BASE_VALUE = False 493 494 # Repository configuration file suggested prefix. If config files 495 # are prefixed with this string, they can be automatically handled 496 # by Entropy. 497 FILENAME_PREFIX = "entropysrv_" 498
499 - def __init__(self, encoding = None):
501 502 @classmethod
503 - def _validate_section(cls, match):
504 """ 505 Reimpemented from BaseConfigParser. 506 """ 507 # a new repository begins 508 groups = match.groups() 509 if not groups: 510 return 511 512 candidate = groups[0] 513 prefix = "server=" 514 if not candidate.startswith(prefix): 515 return 516 candidate = candidate[len(prefix):] 517 if not entropy.tools.validate_repository_id(candidate): 518 return 519 return candidate
520
521 - def base_repository(self):
522 """ 523 Return the base repository, if any, or None. 524 525 @return: the base repository identifier 526 @rtype: string or None 527 """ 528 repositories = self.repositories() 529 base = None 530 for repository_id in repositories: 531 try: 532 p_value = self[repository_id]["base"][0] 533 value = False 534 if p_value.strip().lower() == "true": 535 value = True 536 except KeyError: 537 value = self._DEFAULT_BASE_VALUE 538 if value: 539 base = repository_id 540 break 541 542 if base is None and repositories: 543 base = repositories[0] 544 return base
545
546 - def add(self, repository_id, desc, repo, repo_only, pkg_only, 547 base, enabled = True, exclude_qa = False):
548 """ 549 Add a repository to the repository configuration files directory. 550 Older repository configuration may get overwritten. This method 551 only writes repository configuration in the new .ini format and to 552 /etc/entropy/repositories.conf.d/<filename prefix><repository id>. 553 554 @param repository_id: repository identifier 555 @type repository_id: string 556 @param desc: repository description 557 @type desc: string 558 @param repo: list of "repo=" uris 559 @type repo: list 560 @param repo_only: list of "repo-only=" uris 561 @type repo_only: list 562 @param pkg_only: list of "pkg-only=" uris 563 @type pkg_only: list 564 @param base: True, if this is the base repository 565 @type base: bool 566 @keyword enabled: True, if the repository is enabled 567 @type enabled: bool 568 @keyword exclude_qa: True, if the repository should be excluded from QA 569 @type exclude_qa: bool 570 """ 571 settings = SystemSettings() 572 repo_d_conf = settings.get_setting_dirs_data()['repositories_conf_d'] 573 conf_d_dir, _conf_files_mtime, _skipped_files, _auto_upd = repo_d_conf 574 # as per specifications, enabled config files handled by 575 # Entropy Server (see repositories.conf.d/README) start with 576 # entropysrv_ prefix. 577 base_name = self.FILENAME_PREFIX + repository_id 578 enabled_conf_file = os.path.join(conf_d_dir, base_name) 579 # while disabled config files start with _ 580 disabled_conf_file = os.path.join(conf_d_dir, "_" + base_name) 581 582 self.write(enabled_conf_file, repository_id, desc, repo, repo_only, 583 pkg_only, base, enabled = enabled, exclude_qa = exclude_qa) 584 585 # if any disabled entry file is around, kill it with fire! 586 try: 587 os.remove(disabled_conf_file) 588 except OSError as err: 589 if err.errno != errno.ENOENT: 590 raise 591 592 return True
593
594 - def remove(self, repository_id):
595 """ 596 Remove a repository from the repositories configuration files directory. 597 598 This method only removes repository configuration at 599 /etc/entropy/repositories.conf.d/<filename prefix><repository id>. 600 601 @param repository_id: repository identifier 602 @type repository_id: string 603 @return: True, if success 604 @rtype: bool 605 """ 606 settings = SystemSettings() 607 repo_d_conf = settings.get_setting_dirs_data()['repositories_conf_d'] 608 conf_d_dir, _conf_files_mtime, _skipped_files, _auto_upd = repo_d_conf 609 # as per specifications, enabled config files handled by 610 # Entropy Server (see repositories.conf.d/README) start with 611 # entropysrv_ prefix. 612 base_name = self.FILENAME_PREFIX + repository_id 613 enabled_conf_file = os.path.join(conf_d_dir, base_name) 614 # while disabled config files start with _ 615 disabled_conf_file = os.path.join(conf_d_dir, "_" + base_name) 616 617 accomplished = False 618 try: 619 os.remove(enabled_conf_file) 620 accomplished = True 621 except OSError as err: 622 if err.errno != errno.ENOENT: 623 raise 624 625 # since we want to remove, also drop disabled 626 # config files 627 try: 628 os.remove(disabled_conf_file) 629 accomplished = True 630 except OSError as err: 631 if err.errno != errno.ENOENT: 632 raise 633 634 return accomplished
635
636 - def write(self, path, repository_id, desc, repo, repo_only, 637 pkg_only, base, enabled = True, exclude_qa = False):
638 """ 639 Write the repository configuration to the given file. 640 641 @param path: configuration file to write 642 @type path: string 643 @param repository_id: repository identifier 644 @type repository_id: string 645 @param desc: repository description 646 @type desc: string 647 @param repo: list of "repo=" uris 648 @type repo: list 649 @param repo_only: list of "repo-only=" uris 650 @type repo_only: list 651 @param pkg_only: list of "pkg-only=" uris 652 @type pkg_only: list 653 @param base: True, if this is the base repository, False if not, None 654 if unset. 655 @type base: bool 656 @keyword enabled: True, if the repository is enabled 657 @type enabled: bool 658 @keyword exclude_qa: True, if the repository should be excluded from QA 659 @type exclude_qa: bool 660 """ 661 if enabled: 662 enabled_str = "true" 663 else: 664 enabled_str = "false" 665 666 if exclude_qa: 667 qa_str = "true" 668 else: 669 qa_str = "false" 670 671 if base: 672 base_str = "base = true" 673 elif base is None: 674 base_str = "# base = false" 675 else: 676 base_str = "base = false" 677 678 repos_str = "" 679 for r in repo: 680 repos_str += "repo = %s\n" % (r,) 681 682 repo_only_str = "" 683 for r in repo_only: 684 repo_only_str += "repo-only = %s\n" % (r,) 685 if not repo_only_str: 686 repo_only_str = "# repo-only = " 687 688 pkg_only_str = "" 689 for pkg in pkg_only: 690 pkg_only_str += "pkg-only = %s\n" % (pkg,) 691 if not pkg_only_str: 692 pkg_only_str = "# pkg-only = " 693 694 meta = { 695 "repository_id": repository_id, 696 "desc": desc, 697 "repos": repos_str.rstrip(), 698 "repo_only": repo_only_str.rstrip(), 699 "pkg_only": pkg_only_str.rstrip(), 700 "enabled": enabled_str, 701 "exclude_qa": qa_str, 702 "base": base_str, 703 } 704 705 config = """\ 706 # Repository configuration file automatically generated 707 # by Entropy Server on your behalf. 708 709 [server=%(repository_id)s] 710 %(base)s 711 exclude-qa = %(exclude_qa)s 712 desc = %(desc)s 713 %(repos)s 714 %(repo_only)s 715 %(pkg_only)s 716 enabled = %(enabled)s 717 """ % meta 718 719 entropy.tools.atomic_write(path, config, self._encoding)
720
721 - def repositories(self):
722 """ 723 Return a list of valid parsed repositories. 724 725 A repository is considered valid iff it contains 726 at least "repo". The parse order is preserved. 727 """ 728 required_keys = set(("repo",)) 729 repositories = [] 730 731 for repository_id in self._ordered_sections: 732 repo_data = self[repository_id] 733 remaining = required_keys - set(repo_data.keys()) 734 if not remaining: 735 # then required_keys are there 736 repositories.append(repository_id) 737 738 return repositories
739
740 - def repo(self, repository_id):
741 """ 742 Return the repository push & pull URIs for both packages and 743 repository database. 744 745 @param repository_id: the repository identifier 746 @type repository_id: string 747 @raise KeyError: if repository_id is not found or 748 metadata is not available 749 @return: the repository push & pull URIs. 750 @rtype: list 751 """ 752 return self[repository_id]["repo"]
753
754 - def repo_only(self, repository_id):
755 """ 756 Return the repository push & pull URIs for the repository 757 database only. 758 759 @param repository_id: the repository identifier 760 @type repository_id: string 761 @raise KeyError: if repository_id is not found or 762 metadata is not available 763 @return: the repository push & pull URIs for the repository 764 database only. 765 @rtype: list 766 """ 767 return self[repository_id]["repo-only"]
768
769 - def pkg_only(self, repository_id):
770 """ 771 Return the repository push & pull URIs for the repository only. 772 773 @param repository_id: the repository identifier 774 @type repository_id: string 775 @raise KeyError: if repository_id is not found or 776 metadata is not available 777 @return: the repository push & pull URIs for the packages only. 778 @rtype: list 779 """ 780 return self[repository_id]["pkg-only"]
781
782 - def desc(self, repository_id):
783 """ 784 Return the description of the repository. 785 786 @param repository_id: the repository identifier 787 @type repository_id: string 788 @raise KeyError: if repository_id is not found or 789 metadata is not available 790 @return: the repository description 791 @rtype: string 792 """ 793 return self[repository_id]["desc"][0]
794
795 - def enabled(self, repository_id):
796 """ 797 Return whether the repository is enabled or disabled. 798 799 @param repository_id: the repository identifier 800 @type repository_id: string 801 @return: the repository status 802 @rtype: bool 803 """ 804 try: 805 enabled = self[repository_id]["enabled"][0] 806 return enabled.strip().lower() == "true" 807 except KeyError: 808 return self._DEFAULT_ENABLED_VALUE
809
810 - def exclude_qa(self, repository_id):
811 """ 812 Return whether the repository is excluded from QA. 813 814 @param repository_id: the repository identifier 815 @type repository_id: string 816 @return: the repository QA exclusion status 817 @rtype: bool 818 """ 819 try: 820 exclude = self[repository_id]["exclude-qa"][0] 821 return exclude.strip().lower() == "true" 822 except KeyError: 823 return self._DEFAULT_QA_VALUE
824
825 826 -class ServerSystemSettingsPlugin(SystemSettingsPlugin):
827 828 # List of static server-side repositories that must survive 829 # a repositories metadata reload 830 REPOSITORIES = {} 831
832 - def __init__(self, plugin_id, helper_interface):
833 SystemSettingsPlugin.__init__(self, plugin_id, helper_interface)
834 835 @staticmethod
836 - def server_conf_path():
837 """ 838 Return current server.conf path, this takes into account the current 839 configuration files directory path (which is affected by "root" path 840 changes [default: /]) 841 """ 842 # path to /etc/entropy/server.conf (usually, depends on systemroot) 843 return os.path.join(etpConst['confdir'], "server.conf")
844 845 @classmethod
846 - def analyze_server_repo_string(cls, repostring, product = None):
847 """ 848 Analyze a server repository string (usually contained in server.conf), 849 extracting all the parameters. 850 851 @param repostring: repository string 852 @type repostring: string 853 @keyword product: system product which repository belongs to 854 @rtype: None 855 @return: None 856 """ 857 858 if product is None: 859 product = etpConst['product'] 860 861 data = {} 862 repo_key, repostring = entropy.tools.extract_setting(repostring) 863 if repo_key != "repository": 864 raise AttributeError("invalid repostring passed") 865 866 repo_split = repostring.split("|") 867 if len(repo_split) < 3: 868 raise AttributeError("invalid repostring passed (2)") 869 870 repository_id = repo_split[0].strip() 871 desc = repo_split[1].strip() 872 uris = repo_split[2].strip().split() 873 exclude_qa = False # not supported through server.conf 874 875 repo_mirrors = [] 876 pkg_mirrors = [] 877 for uri in uris: 878 do_pkg = False 879 do_repo = False 880 while True: 881 if uri.startswith("<p>"): 882 do_pkg = True 883 uri = uri[3:] 884 continue 885 if uri.startswith("<r>"): 886 do_repo = True 887 uri = uri[3:] 888 continue 889 break 890 891 if not (do_repo or do_pkg): 892 do_repo = True 893 do_pkg = True 894 if do_repo: 895 repo_mirrors.append(uri) 896 if do_pkg: 897 pkg_mirrors.append(uri) 898 899 return repository_id, cls._generate_repository_metadata( 900 repository_id, desc, repo_mirrors, pkg_mirrors, exclude_qa)
901 902 @classmethod
903 - def _generate_repository_metadata(cls, repository_id, desc, 904 repo_mirrors, pkg_mirrors, 905 exclude_qa):
906 """ 907 Generate the repository metadata given raw information. 908 909 @param repository_id: the repository identifier 910 @type repository_id: string 911 @param desc: repository description 912 @type desc: string 913 @param repo_mirrors: list of repository database mirrors 914 @type repo_mirrors: list 915 @param pkg_mirrors: list of repository packages mirrors 916 @type pkg_mirrors: list 917 @param exclude_qa: exclude from QA checks 918 @type exclude_qa: bool 919 @return: the repository metadata 920 @rtype: dict 921 """ 922 data = {} 923 data['repoid'] = repository_id 924 data['description'] = desc 925 data['pkg_mirrors'] = pkg_mirrors[:] 926 data['repo_mirrors'] = repo_mirrors[:] 927 data['community'] = False 928 data['exclude_qa'] = exclude_qa 929 return data
930
931 - def __generic_parser(self, filepath):
932 """ 933 Internal method. This is the generic file parser here. 934 935 @param filepath: valid path 936 @type filepath: string 937 @return: raw text extracted from file 938 @rtype: list 939 """ 940 return entropy.tools.generic_file_content_parser( 941 filepath, 942 comment_tag = "##", 943 encoding = etpConst['conf_encoding'])
944
945 - def get_updatable_configuration_files(self, repository_id):
946 """ 947 Overridden from SystemSettings. 948 """ 949 files = set() 950 # hope that all the repos get synchronized with respect to 951 # package names moves 952 dep_rewrite_file = Server._get_conf_dep_rewrite_file() 953 dep_blacklist_file = Server._get_conf_dep_blacklist_file() 954 files.add(dep_rewrite_file) 955 files.add(dep_blacklist_file) 956 957 if (repository_id is not None) and \ 958 (repository_id in self._helper.repositories()): 959 960 critical_file = self._helper._get_local_critical_updates_file( 961 repository_id) 962 files.add(critical_file) 963 keywords_file = self._helper._get_local_repository_keywords_file( 964 repository_id) 965 files.add(keywords_file) 966 mask_file = self._helper._get_local_repository_mask_file( 967 repository_id) 968 files.add(mask_file) 969 bl_file = self._helper._get_missing_dependencies_blacklist_file( 970 repository_id) 971 files.add(bl_file) 972 restricted_file = self._helper._get_local_restricted_file( 973 repository_id) 974 files.add(restricted_file) 975 system_mask_file = \ 976 self._helper._get_local_repository_system_mask_file( 977 repository_id) 978 files.add(system_mask_file) 979 980 return files
981
982 - def dep_rewrite_parser(self, sys_set):
983 984 cached = getattr(self, '_mod_rewrite_data', None) 985 if cached is not None: 986 return cached 987 988 data = [] 989 rewrite_file = Server._get_conf_dep_rewrite_file() 990 if not os.path.isfile(rewrite_file): 991 return data 992 rewrite_content = self.__generic_parser(rewrite_file) 993 994 def add_dep_handler(dep_pattern): 995 metadata = {'dep_pattern_str': dep_pattern, 996 'replaces_str': _("added"), 997 'action': "add", 998 'add_what': dep_pattern} 999 return metadata
1000 1001 def remove_dep_handler(dep_pattern, compiled_pattern): 1002 metadata = {'dep_pattern_str': dep_pattern, 1003 'replaces_str': _("removed"), 1004 'action': "remove", 1005 'does_dep_match_func': lambda dep_string: compiled_pattern.match(dep_string) is not None, 1006 'replaces': []} 1007 return metadata
1008 1009 def replace_dep_handler(dep_pattern, compiled_pattern, replaces): 1010 def _do(replace, dep_string): 1011 new_dep_string, number_of_subs_made = \ 1012 compiled_pattern.subn(replace, dep_string) 1013 return new_dep_string, bool(number_of_subs_made) 1014 1015 metadata = {'dep_pattern_str': dep_pattern, 1016 'replaces_str': "=> " + ', '.join(replaces), 1017 'action': "replace", 1018 'does_dep_match_func': lambda dep_string: compiled_pattern.match(dep_string) is not None, 1019 'replaces': replaces, 1020 'do_func': _do} 1021 return metadata 1022 1023 def new_replace_dep_handler(rule): 1024 def _do(_unused, dep_string): 1025 try: 1026 # Should not actually happen as it's tested with does_dep_match_func. 1027 rewriter = entropy.dep.DependencyRewriter([dep_string], rule) 1028 except entropy.dep.WrongRewriteRuleError: 1029 return dep_string, False 1030 1031 rewriter.rewrite() 1032 return rewriter.deps[0], rewriter.changed 1033 1034 def _does_dep_match(dep_string): 1035 try: 1036 rewriter = entropy.dep.DependencyRewriter([dep_string], rule) 1037 except entropy.dep.WrongRewriteRuleError: 1038 return False 1039 1040 rewriter.rewrite() 1041 return rewriter.matched 1042 1043 metadata = {'dep_pattern_str': "rewrite", 1044 # not translated to match the keyword in configuration 1045 'replaces_str': "~> %s" % (rule,), 1046 'action': "replace", 1047 'does_dep_match_func': _does_dep_match, 1048 'replaces': [rule], 1049 'do_func': _do} 1050 return metadata 1051 1052 for line in rewrite_content: 1053 params = line.split() 1054 if len(params) < 2: 1055 continue 1056 1057 if params[0] == "rewrite": 1058 if len(params) < 3: 1059 continue 1060 _unused, pkg_match, rule = line.split(None, 2) 1061 metadata = new_replace_dep_handler(rule) 1062 data.append((pkg_match, metadata)) 1063 continue 1064 1065 pkg_match, pattern, replaces = params[0], params[1], params[2:] 1066 if pattern.startswith("++"): 1067 compiled_pattern = None 1068 pattern = pattern[2:] 1069 if not pattern: 1070 # malformed 1071 continue 1072 else: 1073 try: 1074 compiled_pattern = re.compile(pattern) 1075 except re.error: 1076 # invalid pattern 1077 continue 1078 1079 if compiled_pattern is None: 1080 # this means that user is asking to add dep_pattern 1081 # as a dependency to package 1082 metadata = add_dep_handler(pattern) 1083 elif not replaces: 1084 # this means that user is asking to remove dep_pattern 1085 metadata = remove_dep_handler(pattern, compiled_pattern) 1086 else: 1087 metadata = replace_dep_handler(pattern, compiled_pattern, replaces) 1088 # use this key to make sure to not overwrite similar entries 1089 data.append((pkg_match, metadata)) 1090 1091 self._mod_rewrite_data = data 1092 return data 1093
1094 - def dep_blacklist_parser(self, sys_set):
1095 1096 data = {} 1097 blacklist_file = Server._get_conf_dep_blacklist_file() 1098 if not os.path.isfile(blacklist_file): 1099 return data 1100 blacklist_content = self.__generic_parser(blacklist_file) 1101 1102 for line in blacklist_content: 1103 params = line.split() 1104 if len(params) < 2: 1105 continue 1106 pkg_match, blacklisted_deps = params[0], params[1:] 1107 # use this key to make sure to not overwrite similar entries 1108 obj = data.setdefault(pkg_match, []) 1109 obj.extend(blacklisted_deps) 1110 1111 return data
1112
1113 - def qa_sets_parser(self, sys_set):
1114 1115 data = {} 1116 sets_file = Server._get_conf_qa_sets_file() 1117 if not os.path.isfile(sets_file): 1118 return data 1119 sets_content = self.__generic_parser(sets_file) 1120 1121 for line in sets_content: 1122 params = line.split() 1123 if len(params) < 2: 1124 continue 1125 repo_id, qa_sets = params[0], params[1:] 1126 obj = data.setdefault(repo_id, set()) 1127 obj.update(qa_sets) 1128 1129 return data
1130
1131 - def server_parser(self, sys_set):
1132 """ 1133 Parses Entropy server system configuration file. 1134 1135 @return dict data 1136 """ 1137 srv_plugin_class = ServerSystemSettingsPlugin 1138 server_conf = srv_plugin_class.server_conf_path() 1139 enc = etpConst['conf_encoding'] 1140 1141 try: 1142 with codecs.open(server_conf, "r", encoding=enc) \ 1143 as server_f: 1144 serverconf = [x.strip() for x in server_f.readlines() \ 1145 if x.strip()] 1146 except IOError as err: 1147 if err.errno != errno.ENOENT: 1148 raise 1149 # if file doesn't exist, provide empty 1150 # serverconf list. In this way, we make sure that 1151 # any additional metadata gets added. 1152 # see the for loop iterating through the 1153 # repository identifiers 1154 serverconf = [] 1155 1156 data = { 1157 'repositories': srv_plugin_class.REPOSITORIES.copy(), 1158 'community_mode': False, 1159 'qa_langs': [const_convert_to_unicode("en_US"), 1160 const_convert_to_unicode("C")], 1161 'default_repository_id': const_convert_to_unicode( 1162 etpConst['defaultserverrepositoryid']), 1163 'base_repository_id': None, 1164 'packages_expiration_days': etpConst['packagesexpirationdays'], 1165 'database_file_format': const_convert_to_unicode( 1166 etpConst['etpdatabasefileformat']), 1167 'disabled_eapis': set(), 1168 'broken_revdeps_qa_check': True, 1169 'exp_based_scope': etpConst['expiration_based_scope'], 1170 # disabled by default for now 1171 'nonfree_packages_dir_support': False, 1172 'sync_speed_limit': None, 1173 'weak_package_files': False, 1174 'changelog': True, 1175 'rss': { 1176 'enabled': etpConst['rss-feed'], 1177 'name': const_convert_to_unicode(etpConst['rss-name']), 1178 'light_name': const_convert_to_unicode( 1179 etpConst['rss-light-name']), 1180 'base_url': const_convert_to_unicode(etpConst['rss-base-url']), 1181 'website_url': const_convert_to_unicode( 1182 etpConst['rss-website-url']), 1183 'editor': const_convert_to_unicode( 1184 etpConst['rss-managing-editor']), 1185 'max_entries': etpConst['rss-max-entries'], 1186 'light_max_entries': etpConst['rss-light-max-entries'], 1187 }, 1188 } 1189 1190 fake_instance = self._helper.fake_default_repo 1191 default_repo_changed = False 1192 1193 def _offservrepoid(line, setting): 1194 # NOTE: remove this in future, supported for backward compat. 1195 # NOTE: added for backward and mixed compat. 1196 if default_repo_changed: 1197 return 1198 if not fake_instance: 1199 data['default_repository_id'] = setting.strip()
1200 1201 def _default_repo(line, setting): 1202 if not fake_instance: 1203 data['default_repository_id'] = setting.strip() 1204 default_repo_changed = True 1205 1206 def _exp_days(line, setting): 1207 mydays = setting.strip() 1208 try: 1209 mydays = int(mydays) 1210 data['packages_expiration_days'] = mydays 1211 except ValueError: 1212 return 1213 1214 def _exp_based_scope(line, setting): 1215 exp_opt = entropy.tools.setting_to_bool(setting) 1216 if exp_opt is not None: 1217 data['exp_based_scope'] = exp_opt 1218 1219 def _nf_packages_dir_sup(line, setting): 1220 opt = entropy.tools.setting_to_bool(setting) 1221 if opt is not None: 1222 data['nonfree_packages_dir_support'] = opt 1223 1224 def _disabled_eapis(line, setting): 1225 mydis = setting.strip().split(",") 1226 try: 1227 mydis = [int(x) for x in mydis] 1228 mydis = set([x for x in mydis if x in (1, 2, 3,)]) 1229 except ValueError: 1230 return 1231 if (len(mydis) < 3) and mydis: 1232 data['disabled_eapis'] = mydis 1233 1234 def _server_basic_lang(line, setting): 1235 data['qa_langs'] = setting.strip().split() 1236 1237 def _broken_revdeps_qa(line, setting): 1238 opt = entropy.tools.setting_to_bool(setting) 1239 if opt is not None: 1240 data['broken_revdeps_qa_check'] = opt 1241 1242 def _repository_func(line, setting): 1243 # TODO: deprecate in 2015. repositories.conf.d/ is the 1244 # supported way to define repositories. 1245 try: 1246 repoid, repodata = \ 1247 srv_plugin_class.analyze_server_repo_string( 1248 line, product = sys_set['repositories']['product']) 1249 except AttributeError: 1250 # error parsing string 1251 return 1252 1253 # validate repository id string 1254 if not entropy.tools.validate_repository_id(repoid): 1255 sys.stderr.write("!!! invalid repository id '%s' in '%s'\n" % ( 1256 repoid, srv_plugin_class.server_conf_path())) 1257 return 1258 1259 if repoid in data['repositories']: 1260 # just update mirrors 1261 data['repositories'][repoid]['pkg_mirrors'].extend( 1262 repodata['pkg_mirrors']) 1263 data['repositories'][repoid]['repo_mirrors'].extend( 1264 repodata['repo_mirrors']) 1265 else: 1266 data['repositories'][repoid] = repodata.copy() 1267 1268 # base_repository_id support 1269 if data['base_repository_id'] is None: 1270 data['base_repository_id'] = repoid 1271 1272 def _database_format(line, setting): 1273 if setting in etpConst['etpdatabasesupportedcformats']: 1274 data['database_file_format'] = setting 1275 1276 def _syncspeedlimit(line, setting): 1277 try: 1278 speed_limit = int(setting) 1279 except ValueError: 1280 speed_limit = None 1281 data['sync_speed_limit'] = speed_limit 1282 1283 def _weak_package_files(line, setting): 1284 opt = entropy.tools.setting_to_bool(setting) 1285 if opt is not None: 1286 data['weak_package_files'] = opt 1287 1288 def _changelog(line, setting): 1289 bool_setting = entropy.tools.setting_to_bool(setting) 1290 if bool_setting is not None: 1291 data['changelog'] = bool_setting 1292 1293 def _community_mode(line, setting): 1294 bool_setting = entropy.tools.setting_to_bool(setting) 1295 if bool_setting is not None: 1296 data['community_mode'] = bool_setting 1297 1298 def _rss_feed(line, setting): 1299 bool_setting = entropy.tools.setting_to_bool(setting) 1300 if bool_setting is not None: 1301 data['rss']['enabled'] = bool_setting 1302 1303 def _rss_name(line, setting): 1304 data['rss']['name'] = setting 1305 1306 def _rss_light_name(line, setting): 1307 data['rss']['light_name'] = setting 1308 1309 def _rss_base_url(line, setting): 1310 data['rss']['base_url'] = setting 1311 1312 def _rss_website_url(line, setting): 1313 data['rss']['website_url'] = setting 1314 1315 def _managing_editor(line, setting): 1316 data['rss']['editor'] = setting 1317 1318 def _max_rss_entries(line, setting): 1319 try: 1320 entries = int(setting) 1321 data['rss']['max_entries'] = entries 1322 except (ValueError, IndexError,): 1323 return 1324 1325 def _max_rss_light_entries(line, setting): 1326 try: 1327 entries = int(setting) 1328 data['rss']['light_max_entries'] = entries 1329 except (ValueError, IndexError,): 1330 return 1331 1332 settings_map = { 1333 'officialserverrepositoryid': _offservrepoid, 1334 'default-repository': _default_repo, 1335 'expiration-days': _exp_days, 1336 'community-mode': _community_mode, 1337 'expiration-based-scope': _exp_based_scope, 1338 'nonfree-packages-directory-support': _nf_packages_dir_sup, 1339 'disabled-eapis': _disabled_eapis, 1340 'broken-reverse-deps': _broken_revdeps_qa, 1341 'server-basic-languages': _server_basic_lang, 1342 'repository': _repository_func, 1343 'database-format': _database_format, 1344 # backward compatibility 1345 'sync-speed-limit': _syncspeedlimit, 1346 'syncspeedlimit': _syncspeedlimit, 1347 'weak-package-files': _weak_package_files, 1348 'changelog': _changelog, 1349 'rss-feed': _rss_feed, 1350 'rss-name': _rss_name, 1351 'rss-light-name': _rss_light_name, 1352 'rss-base-url': _rss_base_url, 1353 'rss-website-url': _rss_website_url, 1354 'managing-editor': _managing_editor, 1355 'max-rss-entries': _max_rss_entries, 1356 'max-rss-light-entries': _max_rss_light_entries, 1357 } 1358 1359 for line in serverconf: 1360 1361 key, value = entropy.tools.extract_setting(line) 1362 if key is None: 1363 continue 1364 1365 func = settings_map.get(key) 1366 if func is None: 1367 continue 1368 func(line, value) 1369 1370 # .ini-like file support. 1371 repositories_d_conf = sys_set.get_setting_dirs_data( 1372 )['repositories_conf_d'] 1373 _conf_dir, setting_files, _skipped_files, _upd = repositories_d_conf 1374 candidate_inis = [x for x,y in setting_files] 1375 1376 ini_parser = RepositoryConfigParser(encoding = enc) 1377 try: 1378 ini_parser.read(candidate_inis) 1379 except (IOError, OSError) as err: 1380 sys.stderr.write("Cannot parse %s: %s\n" % ( 1381 " ".join(candidate_inis), 1382 err)) 1383 ini_parser = None 1384 1385 if ini_parser: 1386 repositories = set(data['repositories'].keys()) 1387 ini_repositories = ini_parser.repositories() 1388 if data['base_repository_id'] is None: 1389 # if base_repository_id is not set, then 1390 # take the value of ini config files. 1391 ini_base = ini_parser.base_repository() 1392 if ini_base: 1393 data['base_repository_id'] = ini_base 1394 1395 for ini_repository in ini_repositories: 1396 if ini_repository in repositories: 1397 # double syntax is not supported. 1398 continue 1399 ini_enabled = ini_parser.enabled(ini_repository) 1400 if not ini_enabled: 1401 continue 1402 1403 ini_exclude_qa = ini_parser.exclude_qa(ini_repository) 1404 1405 try: 1406 ini_desc = ini_parser.desc(ini_repository) 1407 except KeyError: 1408 ini_desc = _("No description") 1409 try: 1410 ini_mirrors = ini_parser.repo(ini_repository) 1411 except KeyError: 1412 ini_mirrors = [] 1413 1414 repo_mirrors = [] 1415 pkg_mirrors = [] 1416 repo_mirrors.extend(ini_mirrors) 1417 pkg_mirrors.extend(ini_mirrors) 1418 1419 try: 1420 repo_mirrors.extend(ini_parser.repo_only(ini_repository)) 1421 except KeyError: 1422 pass 1423 try: 1424 pkg_mirrors.extend(ini_parser.pkg_only(ini_repository)) 1425 except KeyError: 1426 pass 1427 1428 repo_data = srv_plugin_class._generate_repository_metadata( 1429 ini_repository, ini_desc, repo_mirrors, pkg_mirrors, 1430 ini_exclude_qa) 1431 data['repositories'][ini_repository] = repo_data 1432 1433 env_community_mode = os.getenv("ETP_COMMUNITY_MODE") 1434 if env_community_mode == "0": 1435 data['community_mode'] = False 1436 elif env_community_mode == "1": 1437 data['community_mode'] = True 1438 1439 # add system database if community repository mode is enabled 1440 if data['community_mode']: 1441 client_repository_id = InstalledPackagesRepository.NAME 1442 1443 mydata = srv_plugin_class._generate_repository_metadata( 1444 client_repository_id, 1445 const_convert_to_unicode( 1446 "Community Repositories System Repository"), 1447 [],[], False) 1448 1449 data['repositories'][client_repository_id] = mydata 1450 srv_plugin_class.REPOSITORIES[client_repository_id] = \ 1451 mydata 1452 # installed packages repository is now the base repository 1453 data['base_repository_id'] = client_repository_id 1454 1455 # expand paths 1456 for repoid in data['repositories']: 1457 srv_plugin_class.extend_repository_metadata( 1458 sys_set, repoid, data['repositories'][repoid]) 1459 1460 # Support for shell variables 1461 shell_repoid = os.getenv('ETP_REPO') 1462 if shell_repoid: 1463 data['default_repository_id'] = shell_repoid 1464 1465 expiration_days = os.getenv('ETP_EXPIRATION_DAYS') 1466 if expiration_days: 1467 try: 1468 expiration_days = int(expiration_days) 1469 data['packages_expiration_days'] = expiration_days 1470 except ValueError: 1471 pass 1472 1473 return data 1474 1475 @staticmethod
1476 - def extend_repository_metadata(system_settings, repository_id, metadata):
1477 """ 1478 Extend server-side Repository metadata dictionary 1479 with information required by Entropy Server. 1480 """ 1481 metadata['repo_basedir'] = os.path.join( 1482 etpConst['entropyworkdir'], 1483 "server", 1484 repository_id) 1485 1486 metadata['packages_dir'] = os.path.join( 1487 etpConst['entropyworkdir'], 1488 "server", 1489 repository_id, 1490 etpConst['packagesrelativepath_basedir'], 1491 etpConst['currentarch']) 1492 1493 metadata['packages_dir_nonfree'] = os.path.join( 1494 etpConst['entropyworkdir'], 1495 "server", 1496 repository_id, 1497 etpConst['packagesrelativepath_basedir_nonfree'], 1498 etpConst['currentarch']) 1499 1500 metadata['packages_dir_restricted'] = os.path.join( 1501 etpConst['entropyworkdir'], 1502 "server", 1503 repository_id, 1504 etpConst['packagesrelativepath_basedir_restricted'], 1505 etpConst['currentarch']) 1506 1507 metadata['store_dir'] = os.path.join( 1508 etpConst['entropyworkdir'], 1509 "server", 1510 repository_id, 1511 "store", 1512 etpConst['currentarch']) 1513 1514 # consider this a base dir 1515 metadata['upload_basedir'] = os.path.join( 1516 etpConst['entropyworkdir'], 1517 "server", 1518 repository_id, 1519 "upload") 1520 1521 metadata['database_dir'] = os.path.join( 1522 etpConst['entropyworkdir'], 1523 "server", 1524 repository_id, 1525 "database", 1526 etpConst['currentarch']) 1527 1528 metadata['remote_repo_basedir'] = os.path.join( 1529 system_settings['repositories']['product'], 1530 repository_id) 1531 1532 metadata['database_remote_path'] = \ 1533 ServerSystemSettingsPlugin.get_repository_remote_path( 1534 system_settings, repository_id) 1535 metadata['override_database_remote_path'] = None
1536 1537 @staticmethod
1538 - def get_repository_remote_path(system_settings, repository_id):
1539 return system_settings['repositories']['product'] + "/" + \ 1540 repository_id + "/database/" + etpConst['currentarch']
1541 1542 @staticmethod
1543 - def set_override_remote_repository(system_settings, repository_id, 1544 override_repository_id):
1545 """ 1546 Used to set an overridden remote path where to push repository 1547 database. This can be used for quickly testing repository changes 1548 without directly overwriting the real repository. 1549 """ 1550 repo_path = ServerSystemSettingsPlugin.get_repository_remote_path( 1551 system_settings, override_repository_id) 1552 1553 sys_settings_plugin_id = \ 1554 etpConst['system_settings_plugins_ids']['server_plugin'] 1555 srv_data = system_settings[sys_settings_plugin_id]['server'] 1556 repo_data = srv_data['repositories'][repository_id] 1557 repo_data['override_database_remote_path'] = repo_path
1558
1559 1560 -class ServerFatscopeSystemSettingsPlugin(SystemSettingsPlugin):
1561
1562 - def repos_parser(self, sys_set):
1563 1564 cached = getattr(self, '_repos_data', None) 1565 if cached is not None: 1566 return cached 1567 1568 data = {} 1569 srv_plug_id = etpConst['system_settings_plugins_ids']['server_plugin'] 1570 # if support is not enabled, don't waste time scanning files 1571 srv_parser_data = sys_set[srv_plug_id]['server'] 1572 if not srv_parser_data['exp_based_scope']: 1573 return data 1574 1575 # get expiration-based packages removal data from config files 1576 for repoid in srv_parser_data['repositories']: 1577 1578 # filter out system repository if community repository 1579 # mode is enabled 1580 if repoid == InstalledPackagesRepository.NAME: 1581 continue 1582 1583 package_ids = set() 1584 exp_fp = self._helper._get_local_exp_based_pkgs_rm_whitelist_file( 1585 repoid) 1586 try: 1587 dbconn = self._helper.open_server_repository( 1588 repoid, just_reading = True) 1589 except RepositoryError: 1590 # ignore 1591 continue 1592 1593 pkgs = [] 1594 if const_file_readable(exp_fp): 1595 # don't worry about the race. 1596 pkgs += entropy.tools.generic_file_content_parser( 1597 exp_fp, encoding = etpConst['conf_encoding']) 1598 if '*' in pkgs: # wildcard support 1599 package_ids.add(-1) 1600 else: 1601 for pkg in pkgs: 1602 package_id, rc_match = dbconn.atomMatch(pkg) 1603 if rc_match: 1604 continue 1605 package_ids.add(package_id) 1606 1607 data[repoid] = package_ids 1608 1609 self._repos_data = data 1610 return data
1611
1612 -class ServerFakeClientSystemSettingsPlugin(SystemSettingsPlugin):
1613
1614 - def fake_cli_parser(self, sys_set):
1615 """ 1616 This is just fake, doesn't bring any new metadata but just tweak 1617 Entropy client ones. 1618 """ 1619 data = {} 1620 srv_plug_id = etpConst['system_settings_plugins_ids']['server_plugin'] 1621 # if support is not enabled, don't waste time scanning files 1622 srv_parser_data = sys_set[srv_plug_id]['server'] 1623 1624 # now setup fake Entropy Client repositories, so that Entropy Server 1625 # can use Entropy Client interfaces transparently 1626 srv_repodata = srv_parser_data['repositories'] 1627 cli_repodata = sys_set['repositories'] 1628 # remove unavailable server repos in client metadata first 1629 cli_repodata['available'].clear() 1630 1631 for repoid, repo_data in srv_repodata.items(): 1632 1633 try: 1634 # we must skip the repository validation because 1635 # repositories have been already validated. 1636 # moreover, the system repository_id might not be 1637 # valid (__system__). But still, this is the wanted 1638 # behaviour. 1639 xxx, my_data = sys_set._analyze_client_repo_string( 1640 "repository = %s|%s|http://--fake--|http://--fake--" \ 1641 % (repoid, repo_data['description'],), 1642 _skip_repository_validation=True) 1643 except AttributeError as err: 1644 # yeah, at least let stderr know. 1645 sys.stderr.write(repr(err) + "\n") 1646 continue # sorry! 1647 1648 my_data['repoid'] = repoid 1649 if '__temporary__' in repo_data: 1650 # fake repositories, temp ones 1651 # can't go into Entropy Client, they miss 1652 # 'database_dir' and other metadata 1653 my_data['dbpath'] = None 1654 my_data['__temporary__'] = repo_data['__temporary__'] 1655 my_data['dbrevision'] = 0 1656 else: 1657 my_data['dbpath'] = self._helper._get_local_repository_dir( 1658 repoid) 1659 my_data['dbrevision'] = \ 1660 self._helper.local_repository_revision( 1661 repoid) 1662 cli_repodata['available'][repoid] = my_data 1663 1664 cli_repodata['default_repository'] = \ 1665 srv_parser_data['default_repository_id'] 1666 1667 del cli_repodata['order'][:] 1668 if srv_parser_data['base_repository_id'] is not None: 1669 cli_repodata['order'].append(srv_parser_data['base_repository_id']) 1670 for repoid in sorted(srv_repodata): 1671 if repoid not in cli_repodata['order']: 1672 cli_repodata['order'].append(repoid) 1673 1674 return data
1675
1676 -class ServerQAInterfacePlugin(QAInterfacePlugin):
1677
1678 - def __init__(self, entropy_server_instance):
1679 self._server = entropy_server_instance
1680
1681 - def __check_package_using_spm(self, package_path):
1682 1683 spm_class = get_spm_class() 1684 spm_rc, spm_msg = spm_class.execute_qa_tests(package_path) 1685 1686 if spm_rc == 0: 1687 return True 1688 sys.stderr.write("QA Error: " + spm_msg + "\n") 1689 sys.stderr.flush() 1690 return False
1691
1692 - def __extract_edb_analyze_metadata(self, package_path):
1693 1694 def _is_supported(keywords): 1695 for arch in etpConst['keywords']: 1696 if arch in keywords: 1697 return True 1698 return False
1699 1700 tmp_fd, tmp_f = const_mkstemp(prefix = 'entropy.server') 1701 dbc = None 1702 try: 1703 found_edb = entropy.tools.dump_entropy_metadata(package_path, tmp_f) 1704 if not found_edb: 1705 return False 1706 dbc = self._server._open_temp_repository("test", temp_file = tmp_f, 1707 initialize = False) 1708 for package_id in dbc.listAllPackageIds(): 1709 # NOTE: content is tested in entropy.qa builtin package test 1710 # test content safety 1711 dbc.retrieveContentSafety(package_id) 1712 # test keywords 1713 keywords = dbc.retrieveKeywords(package_id) 1714 if not _is_supported(keywords): 1715 atom = dbc.retrieveAtom(package_id) 1716 # big PHAT warning !! 1717 self._server.output(darkred("~"*40), level = "warning") 1718 self._server.output("[%s, %s] %s" % ( 1719 brown(os.path.basename(package_path)), teal(atom), 1720 purple(_("package has no keyword set, it will be masked !"))), 1721 level = "warning", header = darkred(" !!! ")) 1722 self._server.output(darkred("~"*40), level = "warning") 1723 time.sleep(10) 1724 finally: 1725 if dbc is not None: 1726 dbc.close() 1727 os.close(tmp_fd) 1728 1729 return True
1730
1731 - def get_tests(self):
1732 return [self.__check_package_using_spm, 1733 self.__extract_edb_analyze_metadata]
1734
1735 - def get_id(self):
1736 return SERVER_QA_PLUGIN
1737
1738 1739 -class ServerConfigurationFiles(ConfigurationFiles):
1740 1741 """ 1742 Subclass Entropy Client version in order to return 1743 our repository identifiers 1744 """ 1745 1746 @property
1747 - def _repository_ids(self):
1748 """ 1749 Return a the list of repository identifiers the object 1750 is using. 1751 """ 1752 return self._entropy.repositories()
1753
1754 1755 -class Server(Client):
1756 1757 # Entropy Server cache directory, mainly used for storing commit changes 1758 CACHE_DIR = os.path.join(etpConst['entropyworkdir'], "server_cache") 1759 1760 # SystemSettings class variables 1761 SYSTEM_SETTINGS_PLG_ID = etpConst['system_settings_plugins_ids']['server_plugin'] 1762 1763 # Make possible to disable tree updates completely. 1764 _inhibit_treeupdates = False 1765
1766 - def init_singleton(self, default_repository = None, save_repository = False, 1767 fake_default_repo = False, fake_default_repo_id = None, 1768 fake_default_repo_desc = None, handle_uninitialized = True, 1769 **kwargs):
1770 1771 self._indexing = False 1772 1773 # initialize Entropy Client superclass 1774 if "installed_repo" not in kwargs: 1775 kwargs["installed_repo"] = False 1776 if "repo_validation" not in kwargs: 1777 kwargs["repo_validation"] = False 1778 Client.init_singleton(self, 1779 indexing = self._indexing, 1780 **kwargs 1781 ) 1782 1783 if fake_default_repo_desc is None: 1784 fake_default_repo_desc = 'this is a fake repository' 1785 self.__instance_destroyed = False 1786 1787 # settings 1788 self._memory_db_srv_instances = {} 1789 self._treeupdates_repos = set() 1790 self._server_dbcache = {} 1791 etpSys['serverside'] = True 1792 self.fake_default_repo = fake_default_repo 1793 self.fake_default_repo_id = fake_default_repo_id 1794 self.Mirrors = None 1795 self._settings_to_backup = [] 1796 self._save_repository = save_repository 1797 self._sync_lock_cache = set() 1798 1799 self.sys_settings_fake_cli_plugin_id = \ 1800 etpConst['system_settings_plugins_ids']['server_plugin_fake_client'] 1801 self.sys_settings_fatscope_plugin_id = \ 1802 etpConst['system_settings_plugins_ids']['server_plugin_fatscope'] 1803 1804 # create our SystemSettings plugin 1805 with self._settings: 1806 self.sys_settings_plugin = ServerSystemSettingsPlugin( 1807 Server.SYSTEM_SETTINGS_PLG_ID, self) 1808 self._settings.add_plugin(self.sys_settings_plugin) 1809 1810 # Fatscope support SystemSettings plugin 1811 self.sys_settings_fatscope_plugin = \ 1812 ServerFatscopeSystemSettingsPlugin( 1813 self.sys_settings_fatscope_plugin_id, self) 1814 self._settings.add_plugin(self.sys_settings_fatscope_plugin) 1815 1816 # Fatscope support SystemSettings plugin 1817 self.sys_settings_fake_cli_plugin = \ 1818 ServerFakeClientSystemSettingsPlugin( 1819 self.sys_settings_fake_cli_plugin_id, self) 1820 self._settings.add_plugin(self.sys_settings_fake_cli_plugin) 1821 1822 # setup fake repository 1823 if fake_default_repo: 1824 default_repository = fake_default_repo_id 1825 self._init_generic_memory_server_repository( 1826 fake_default_repo_id, 1827 fake_default_repo_desc, set_as_default = True) 1828 1829 srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server'] 1830 self._repository = default_repository 1831 if self._repository is None: 1832 self._repository = srv_set['default_repository_id'] 1833 1834 if not fake_default_repo: 1835 if self._repository in srv_set['repositories']: 1836 try: 1837 self._ensure_paths(self._repository) 1838 except OSError as err: 1839 if err.errno != errno.EACCES: 1840 raise 1841 # sigh, ignore during init 1842 1843 # if repository is still None, fallback to internal 1844 # fake repository. This way Entropy Server will work 1845 # out of the box without any server.conf tweak 1846 # (and eit bashcomp is happy) 1847 if self._repository is None: 1848 repository_id = "__builtin__" 1849 self._init_generic_memory_server_repository( 1850 repository_id, "Built-in fallback fake repository", 1851 set_as_default=True) 1852 self._repository = repository_id 1853 1854 if self._repository not in srv_set['repositories']: 1855 raise PermissionDenied("PermissionDenied: %s %s" % ( 1856 self._repository, 1857 _("repository not configured"), 1858 ) 1859 ) 1860 if InstalledPackagesRepository.NAME == self._repository: 1861 raise PermissionDenied("PermissionDenied: %s %s" % ( 1862 InstalledPackagesRepository.NAME, 1863 _("protected repository id, can't use this, sorry dude..."), 1864 ) 1865 ) 1866 1867 self.switch_default_repository( 1868 self._repository, handle_uninitialized=handle_uninitialized)
1869
1870 - def destroy(self, _from_shutdown = False):
1871 """ 1872 Destroy this singleton instance. 1873 """ 1874 self.__instance_destroyed = True 1875 Client.close_repositories(self, mask_clear = False) 1876 Client.destroy(self, _from_shutdown = _from_shutdown) 1877 1878 if not _from_shutdown: 1879 plug_id2 = self.sys_settings_fake_cli_plugin_id 1880 plug_id1 = self.sys_settings_fatscope_plugin_id 1881 plug_id = Server.SYSTEM_SETTINGS_PLG_ID 1882 # reverse insert order 1883 plugs = [plug_id2, plug_id1, plug_id] 1884 for plug in plugs: 1885 if plug is None: 1886 continue 1887 if not self._settings.has_plugin(plug): 1888 continue 1889 self._settings.remove_plugin(plug) 1890 1891 self.close_repositories()
1892 1893 @property
1894 - def _cacher(self):
1895 """ 1896 Return an EntropyCacher object instance. 1897 """ 1898 return EntropyCacher()
1899
1900 - def is_destroyed(self):
1901 """ 1902 Return whether the singleton instance is destroyed. 1903 """ 1904 return self.__instance_destroyed
1905
1906 - def _cache_prefix(self, caller):
1907 """ 1908 Generate a cache object key prefix to use with EntropyCacher. 1909 1910 @param caller: a custom function caller name 1911 @type caller: string 1912 @return: the cache prefix 1913 @rtype: string 1914 """ 1915 return "%s/%s/%s" % ( 1916 __name__, self.__class__.__name__, caller)
1917
1918 - def _get_branch_from_download_relative_uri(self, db_download_uri):
1919 return db_download_uri.split("/")[2]
1920
1921 - def _swap_branch_in_download_relative_uri(self, new_branch, 1922 db_download_uri):
1923 cur_branch = self._get_branch_from_download_relative_uri( 1924 db_download_uri) 1925 return db_download_uri.replace("/%s/" % (cur_branch,), 1926 "/%s/" % (new_branch,))
1927
1928 - def _get_basedir_pkg_listing(self, base_dir, extension, branch = None):
1929 1930 pkgs_dir_types = set(self._get_pkg_dir_names()) 1931 basedir_raw_content = [] 1932 entropy.tools.recursive_directory_relative_listing( 1933 basedir_raw_content, base_dir) 1934 1935 pkg_ext = extension 1936 pkg_list = [x for x in basedir_raw_content if x.endswith(pkg_ext)] 1937 pkg_list = [x for x in pkg_list if \ 1938 x.split(os.path.sep)[0] in pkgs_dir_types] 1939 1940 if branch is not None: 1941 branch_extractor = \ 1942 self._get_branch_from_download_relative_uri 1943 pkg_list = [x for x in pkg_list if branch_extractor(x) == branch] 1944 1945 return pkg_list
1946
1947 - def _get_pkg_dir_names(self):
1948 return [etpConst['packagesrelativepath_basedir'], 1949 etpConst['packagesrelativepath_basedir_nonfree'], 1950 etpConst['packagesrelativepath_basedir_restricted']]
1951
1952 - def _get_remote_repository_relative_path(self, repository_id):
1953 srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server'] 1954 return srv_set['repositories'][repository_id]['database_remote_path']
1955
1956 - def _get_override_remote_repository_relative_path(self, repository_id):
1957 srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server'] 1958 repo_data = srv_set['repositories'][repository_id] 1959 return repo_data['override_database_remote_path']
1960
1961 - def _get_local_repository_file(self, repository_id, branch = None):
1962 return os.path.join(self._get_local_repository_dir(repository_id, 1963 branch = branch), etpConst['etpdatabasefile'])
1964
1965 - def _get_local_store_directory(self, repository_id):
1966 srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server'] 1967 return srv_set['repositories'][repository_id]['store_dir']
1968
1969 - def _get_local_upload_directory(self, repository_id):
1970 srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server'] 1971 return srv_set['repositories'][repository_id]['upload_basedir']
1972
1973 - def _get_local_repository_base_directory(self, repository_id):
1974 srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server'] 1975 return srv_set['repositories'][repository_id]['repo_basedir']
1976
1977 - def _get_local_repository_taint_file(self, repository_id, branch = None):
1978 return os.path.join(self._get_local_repository_dir(repository_id, 1979 branch = branch), etpConst['etpdatabasetaintfile'])
1980
1981 - def _get_local_repository_revision_file(self, repository_id, branch = None):
1982 return os.path.join( 1983 self._get_local_repository_dir(repository_id, branch = branch), 1984 etpConst['etpdatabaserevisionfile'])
1985
1986 - def _get_local_repository_timestamp_file(self, repository_id, 1987 branch = None):
1988 return os.path.join(self._get_local_repository_dir(repository_id, 1989 branch = branch), etpConst['etpdatabasetimestampfile'])
1990
1991 - def _get_local_repository_mask_file(self, repository_id, branch = None):
1992 return os.path.join(self._get_local_repository_dir(repository_id, 1993 branch = branch), etpConst['etpdatabasemaskfile'])
1994
1995 - def _get_local_repository_system_mask_file(self, repository_id, 1996 branch = None):
1997 return os.path.join(self._get_local_repository_dir(repository_id, 1998 branch = branch), etpConst['etpdatabasesytemmaskfile'])
1999
2000 - def _get_local_repository_licensewhitelist_file(self, repository_id, 2001 branch = None):
2002 return os.path.join(self._get_local_repository_dir(repository_id, 2003 branch = branch), etpConst['etpdatabaselicwhitelistfile'])
2004
2005 - def _get_local_repository_mirrors_file(self, repository_id, branch = None):
2006 return os.path.join(self._get_local_repository_dir(repository_id, 2007 branch = branch), etpConst['etpdatabasemirrorsfile'])
2008
2009 - def _get_local_repository_fallback_mirrors_file(self, repository_id, 2010 branch = None):
2011 return os.path.join(self._get_local_repository_dir(repository_id, 2012 branch = branch), etpConst['etpdatabasefallbackmirrorsfile'])
2013
2014 - def _get_local_repository_rss_file(self, repository_id, branch = None):
2015 srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server'] 2016 return os.path.join(self._get_local_repository_dir(repository_id, 2017 branch = branch), srv_set['rss']['name'])
2018
2019 - def _get_local_repository_changelog_file(self, repository_id, 2020 branch = None):
2021 srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server'] 2022 return os.path.join(self._get_local_repository_dir(repository_id, 2023 branch = branch), etpConst['changelog_filename'])
2024
2025 - def _get_local_repository_compressed_changelog_file(self, repository_id, 2026 branch = None):
2027 srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server'] 2028 return os.path.join(self._get_local_repository_dir(repository_id, 2029 branch = branch), etpConst['changelog_filename_compressed'])
2030
2031 - def _get_local_repository_rsslight_file(self, repository_id, branch = None):
2032 srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server'] 2033 return os.path.join(self._get_local_repository_dir(repository_id, 2034 branch = branch), srv_set['rss']['light_name'])
2035
2036 - def _get_local_repository_notice_board_file(self, repository_id, 2037 branch = None):
2038 return os.path.join(self._get_local_repository_dir(repository_id, 2039 branch = branch), etpConst['rss-notice-board'])
2040
2041 - def _get_local_repository_treeupdates_file(self, repository_id, 2042 branch = None):
2043 return os.path.join(self._get_local_repository_dir(repository_id, 2044 branch = branch), etpConst['etpdatabaseupdatefile'])
2045
2046 - def _get_local_repository_compressed_metafiles_file(self, repository_id, 2047 branch = None):
2048 return os.path.join(self._get_local_repository_dir(repository_id, 2049 branch = branch), etpConst['etpdatabasemetafilesfile'])
2050
2051 - def _get_local_repository_metafiles_not_found_file(self, repository_id, 2052 branch = None):
2053 return os.path.join(self._get_local_repository_dir(repository_id, 2054 branch = branch), etpConst['etpdatabasemetafilesnotfound'])
2055
2056 - def _get_local_repository_gpg_signature_file(self, repository_id, 2057 branch = None):
2058 return os.path.join(self._get_local_repository_dir(repository_id, 2059 branch = branch), etpConst['etpdatabasegpgfile'])
2060
2061 - def _get_local_exp_based_pkgs_rm_whitelist_file(self, repository_id, 2062 branch = None):
2063 return os.path.join(self._get_local_repository_dir(repository_id, 2064 branch = branch), etpConst['etpdatabaseexpbasedpkgsrm'])
2065
2066 - def _get_local_pkglist_file(self, repository_id, branch = None):
2067 return os.path.join(self._get_local_repository_dir(repository_id, 2068 branch = branch), etpConst['etpdatabasepkglist'])
2069
2070 - def _get_local_extra_pkglist_file(self, repository_id, branch = None):
2071 return os.path.join(self._get_local_repository_dir(repository_id, 2072 branch = branch), etpConst['etpdatabaseextrapkglist'])
2073
2074 - def _get_local_database_sets_dir(self, repository_id, branch = None):
2075 return os.path.join(self._get_local_repository_dir(repository_id, 2076 branch = branch), etpConst['confsetsdirname'])
2077
2078 - def _get_local_post_branch_mig_script(self, repository_id, branch = None):
2079 return os.path.join(self._get_local_repository_dir(repository_id, 2080 branch = branch), etpConst['etp_post_branch_hop_script'])
2081
2082 - def _get_local_post_branch_upg_script(self, repository_id, branch = None):
2083 return os.path.join(self._get_local_repository_dir(repository_id, 2084 branch = branch), etpConst['etp_post_branch_upgrade_script'])
2085
2086 - def _get_local_post_repo_update_script(self, repository_id, branch = None):
2087 return os.path.join(self._get_local_repository_dir(repository_id, 2088 branch = branch), etpConst['etp_post_repo_update_script'])
2089
2090 - def _get_local_critical_updates_file(self, repository_id, branch = None):
2091 return os.path.join(self._get_local_repository_dir(repository_id, 2092 branch = branch), etpConst['etpdatabasecriticalfile'])
2093
2094 - def _get_local_restricted_file(self, repository_id, branch = None):
2095 return os.path.join(self._get_local_repository_dir(repository_id, 2096 branch = branch), etpConst['etpdatabaserestrictedfile'])
2097
2098 - def _get_local_repository_keywords_file(self, repository_id, branch = None):
2099 return os.path.join(self._get_local_repository_dir(repository_id, 2100 branch = branch), etpConst['etpdatabasekeywordsfile'])
2101
2102 - def _get_local_repository_webserv_file(self, repository_id, branch = None):
2103 return os.path.join(self._get_local_repository_dir(repository_id, 2104 branch = branch), etpConst['etpdatabasewebservicesfile'])
2105
2106 - def _get_local_repository_dir(self, repository_id, branch = None):
2107 srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server'] 2108 if branch is None: 2109 branch = self._settings['repositories']['branch'] 2110 return os.path.join( 2111 srv_set['repositories'][repository_id]['database_dir'], branch)
2112
2113 - def _get_missing_dependencies_blacklist_file(self, repository_id, 2114 branch = None):
2115 srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server'] 2116 if branch is None: 2117 branch = self._settings['repositories']['branch'] 2118 return os.path.join( 2119 srv_set['repositories'][repository_id]['database_dir'], 2120 branch, etpConst['etpdatabasemissingdepsblfile'])
2121
2122 - def _get_repository_lockfile(self, repository_id):
2123 return os.path.join(self._get_local_repository_dir(repository_id), 2124 etpConst['etpdatabaselockfile'])
2125
2126 - def _get_repository_download_lockfile(self, repository_id):
2127 return os.path.join(self._get_local_repository_dir(repository_id), 2128 etpConst['etpdatabasedownloadlockfile'])
2129
2130 - def _create_local_repository_download_lockfile(self, repository_id):
2131 lock_file = self._get_repository_download_lockfile(repository_id) 2132 enc = etpConst['conf_encoding'] 2133 with codecs.open(lock_file, "w", encoding=enc) as f_lock: 2134 f_lock.write("download locked")
2135
2136 - def _create_local_repository_lockfile(self, repository_id):
2137 lock_file = self._get_repository_lockfile(repository_id) 2138 enc = etpConst['conf_encoding'] 2139 with codecs.open(lock_file, "w", encoding=enc) as f_lock: 2140 f_lock.write("database locked")
2141
2142 - def _remove_local_repository_lockfile(self, repository_id):
2143 lock_file = self._get_repository_lockfile(repository_id) 2144 try: 2145 os.remove(lock_file) 2146 except OSError: 2147 pass
2148
2149 - def _remove_local_repository_download_lockfile(self, repository_id):
2150 lock_file = self._get_repository_download_lockfile(repository_id) 2151 try: 2152 os.remove(lock_file) 2153 except OSError: 2154 pass
2155 2156 @staticmethod
2158 packages_dir = SystemSettings.packages_config_directory() 2159 return os.path.join(packages_dir, "packages.server.dep_rewrite")
2160 2161 @staticmethod
2163 packages_dir = SystemSettings.packages_config_directory() 2164 return os.path.join(packages_dir, "packages.server.dep_blacklist")
2165 2166 @staticmethod
2168 packages_dir = SystemSettings.packages_config_directory() 2169 return os.path.join(packages_dir, "packages.server.sets")
2170
2171 - def complete_remote_package_relative_path(self, pkg_rel_url, repository_id):
2172 srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server'] 2173 return os.path.join( 2174 srv_set['repositories'][repository_id]['remote_repo_basedir'], 2175 pkg_rel_url)
2176
2177 - def complete_local_upload_package_path(self, pkg_rel_url, repository_id):
2178 srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server'] 2179 return os.path.join( 2180 srv_set['repositories'][repository_id]['upload_basedir'], 2181 pkg_rel_url)
2182
2183 - def complete_local_package_path(self, pkg_rel_url, repository_id):
2184 srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server'] 2185 return os.path.join( 2186 srv_set['repositories'][repository_id]['repo_basedir'], 2187 pkg_rel_url)
2188
2189 - def remote_repository_mirrors(self, repository_id):
2190 """ 2191 Return a list of remote repository mirrors (database) for given 2192 repository. 2193 2194 @param repository_id: repository identifier 2195 @type repository_id: string 2196 @return: list of available repository mirrors 2197 @rtype: list 2198 @raise KeyError: if repository_id is invalid 2199 """ 2200 srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server'] 2201 return srv_set['repositories'][repository_id]['repo_mirrors'][:]
2202
2203 - def remote_packages_mirrors(self, repository_id):
2204 """ 2205 Return a list of remote packages mirrors (packages) for given 2206 repository. 2207 2208 @param repository_id: repository identifier 2209 @type repository_id: string 2210 @return: list of available packages mirrors 2211 @rtype: list 2212 @raise KeyError: if repository_id is invalid 2213 """ 2214 srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server'] 2215 return srv_set['repositories'][repository_id]['pkg_mirrors'][:]
2216
2217 - def local_repository_revision(self, repository_id):
2218 """ 2219 Return local repository revision. 2220 2221 @param repository_id: repository identifier 2222 @type repository_id: string 2223 @return: the actual repository revision 2224 @rtype: int 2225 """ 2226 dbrev_file = self._get_local_repository_revision_file(repository_id) 2227 if not os.path.isfile(dbrev_file): 2228 return 0 2229 2230 enc = etpConst['conf_encoding'] 2231 with codecs.open(dbrev_file, "r", encoding=enc) as f_rev: 2232 rev = f_rev.readline().strip() 2233 try: 2234 rev = int(rev) 2235 except ValueError: 2236 self.output( 2237 "[%s] %s: %s - %s" % ( 2238 darkgreen(repository_id), 2239 blue(_("invalid repository revision")), 2240 bold(rev), 2241 blue(_("defaulting to 0")), 2242 ), 2243 importance = 2, 2244 level = "error", 2245 header = darkred(" !!! ") 2246 ) 2247 rev = 0 2248 return rev
2249
2250 - def remote_repository_revision(self, repository_id):
2251 """ 2252 Return the highest repository revision available on mirrors for 2253 given repository. 2254 2255 @param repository_id: repository identifier 2256 @type repository_id: string 2257 @return: remote repository revision 2258 @rtype: int 2259 """ 2260 repo_status = self.Mirrors.remote_repository_status(repository_id) 2261 remote_status = list(repo_status.items()) 2262 if not [x for x in remote_status if x[1]]: 2263 return 0 2264 return max([x[1] for x in remote_status])
2265
2266 - def repositories(self):
2267 """ 2268 Return a list of available Entropy Server repositories. 2269 2270 @return: list of available Entropy Server repositories 2271 @rtype: list 2272 """ 2273 return self._enabled_repos
2274 2275 @property
2276 - def _enabled_repos(self):
2277 """ 2278 Monkey-patched Entropy Client property provided for compatibility. 2279 """ 2280 try: 2281 srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server'] 2282 except KeyError: 2283 # this happens during the init phase, ignore 2284 return [] 2285 else: 2286 return sorted(srv_set['repositories'])
2287
2288 - def qa_repositories(self):
2289 """ 2290 Return a list of QA-testable available Entropy Server repositories. 2291 2292 @return: list of QA-testable available Entropy Server repositories 2293 @rtype: list 2294 """ 2295 srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server'] 2296 repos = srv_set['repositories'] 2297 return sorted([x for x, y in repos.items() if not y['exclude_qa']])
2298
2299 - def repository(self):
2300 """ 2301 Return the current repository marked as default. 2302 """ 2303 return self._repository
2304
2305 - def QA(self):
2306 """ 2307 Get Entropy QA Interface instance. 2308 2309 @return: Entropy QA Interface instance 2310 @rtype: entropy.qa.QAInterface instance 2311 """ 2312 qa_plugin = ServerQAInterfacePlugin(self) 2313 qa = Client.QA(self) 2314 qa.add_plugin(qa_plugin) 2315 return qa
2316
2317 - def Spm(self):
2318 """ 2319 Get Source Package Manager interface instance. 2320 2321 #@return: Source Package Manager interface instance 2322 @rtype: entropy.spm.plugins.skel.SpmPlugin based instance 2323 """ 2324 return get_spm(self)
2325
2326 - def Spm_class(self):
2327 """ 2328 Get Source Package Manager interface class. 2329 """ 2330 return get_spm_class()
2331
2332 - def ConfigurationUpdates(self):
2333 """ 2334 Return Entropy Configuration File Updates management object. 2335 """ 2336 return ConfigurationUpdates( 2337 self, _config_class=ServerConfigurationFiles)
2338
2339 - def Transceiver(self, uri):
2340 """ 2341 Get EntropyTransceiver interface instance. 2342 2343 @param uri: EntropyTransceiver URI 2344 @type uri: string 2345 @return: EntropyTransceiver instance 2346 @rtype: entropy.transceivers.EntropyTransceiver 2347 """ 2348 txc = EntropyTransceiver(uri) 2349 txc.set_output_interface(self) 2350 return txc
2351
2352 - def _sync_package_sets(self, entropy_repository):
2353 """ 2354 Synchronize repository package sets. 2355 2356 @param entropy_repository: EntropyRepositoryBase object 2357 @type entropy_repository: entropy.db.skel.EntropyRepositoryBase 2358 """ 2359 repository_id = entropy_repository.repository_id() 2360 self.output( 2361 "[%s|%s] %s" % ( 2362 blue(repository_id), 2363 red(_("repository")), 2364 blue(_("syncing package sets")), 2365 ), 2366 importance = 1, 2367 level = "info", 2368 header = brown(" @@ ") 2369 ) 2370 cur_sets = entropy_repository.retrievePackageSets() 2371 sys_sets = self._get_configured_package_sets(repository_id) 2372 if cur_sets != sys_sets: 2373 self._update_package_sets(repository_id, entropy_repository) 2374 # NOTE: this is called by the commit hook plugin, keep no_plugins=True! 2375 entropy_repository.commit(no_plugins = True)
2376
2377 - def sets_available(self, *args, **kwargs):
2378 sets = Client.Sets(self) 2379 return sets.available(*args, **kwargs)
2380
2381 - def sets_search(self, *args, **kwargs):
2382 sets = Client.Sets(self) 2383 return sets.search(*args, **kwargs)
2384
2385 - def sets_match(self, *args, **kwargs):
2386 sets = Client.Sets(self) 2387 return sets.match(*args, **kwargs)
2388
2389 - def atom_match(self, *args, **kwargs):
2390 # disable masked packages for server-side repos 2391 kwargs['mask_filter'] = False 2392 return Client.atom_match(self, *args, **kwargs)
2393
2394 - def _match_packages(self, repository_id, packages):
2395 2396 dbconn = self.open_server_repository(repository_id, read_only = True, 2397 no_upload = True) 2398 if ("world" in packages) or not packages: 2399 return dbconn.listAllPackageIds(), True 2400 else: 2401 package_ids = set() 2402 for package in packages: 2403 matches = dbconn.atomMatch(package, multiMatch = True) 2404 if matches[1] == 0: 2405 package_ids |= matches[0] 2406 else: 2407 mytxt = "%s: %s: %s" % ( 2408 red(_("Attention")), 2409 blue(_("cannot match")), 2410 bold(package), 2411 ) 2412 self.output( 2413 mytxt, 2414 importance = 1, 2415 level = "warning", 2416 header = darkred(" !!! ") 2417 ) 2418 return package_ids, False
2419
2420 - def mask_packages(self, repository_id, packages):
2421 """ 2422 Mask given package dependencies for given repository, if any (otherwise 2423 use default one). 2424 2425 @param repository_id: repository identifier 2426 @type repository_id: string 2427 @param packages: list of package dependency strings 2428 @type packages: list 2429 @return: mask status, True if ok, False if not 2430 @rtype: bool 2431 """ 2432 mask_file = self._get_local_repository_mask_file(repository_id) 2433 current_packages = [] 2434 2435 if const_file_readable(mask_file): 2436 # don't worry about the race. 2437 current_packages += entropy.tools.generic_file_content_parser( 2438 mask_file, comment_tag = "##", filter_comments = False, 2439 encoding = etpConst['conf_encoding']) 2440 # this is untrusted input, it's fine because that config file is 2441 # untrusted too 2442 current_packages.extend(packages) 2443 2444 mask_file_tmp = mask_file + ".mask_packages_tmp" 2445 enc = etpConst['conf_encoding'] 2446 with codecs.open(mask_file_tmp, "w", encoding=enc) as mask_f: 2447 for package in current_packages: 2448 mask_f.write(package + "\n") 2449 2450 os.rename(mask_file_tmp, mask_file) 2451 2452 return True
2453
2454 - def unmask_packages(self, repository_id, packages):
2455 """ 2456 Unmask given package dependencies for given repository, if any 2457 (otherwise use default one). 2458 2459 @param repository_id: repository identifier 2460 @type repository_id: string 2461 @param packages: list of package dependency strings 2462 @type packages: list 2463 @return: mask status, True if ok, False if not 2464 @rtype: bool 2465 """ 2466 mask_file = self._get_local_repository_mask_file(repository_id) 2467 current_packages = [] 2468 2469 if const_file_readable(mask_file): 2470 # don't worry about the race. 2471 current_packages += entropy.tools.generic_file_content_parser( 2472 mask_file, comment_tag = "##", filter_comments = False, 2473 encoding = etpConst['conf_encoding']) 2474 2475 def mask_filter(package): 2476 if package.startswith("#"): 2477 # comment, always valid 2478 return True 2479 in_file_pkg_match = self.atom_match(package) 2480 for req_package in packages: 2481 if package == req_package: 2482 # of course remove if it's equal 2483 return False 2484 req_package_match = self.atom_match(req_package) 2485 if req_package_match == in_file_pkg_match: 2486 # drop it, they point to the same package match 2487 return False 2488 return True
2489 2490 current_packages = list(filter(mask_filter, current_packages)) 2491 2492 mask_file_tmp = mask_file + ".mask_packages_tmp" 2493 enc = etpConst['conf_encoding'] 2494 with codecs.open(mask_file_tmp, "w", encoding=enc) as mask_f: 2495 for package in current_packages: 2496 mask_f.write(package + "\n") 2497 2498 os.rename(mask_file_tmp, mask_file) 2499 2500 return True
2501
2502 - def initialize_repository(self, repository_id, ask = True):
2503 """ 2504 Initialize (and wipe all data!) given repository to empty status. 2505 2506 @param repository_id: repository identifier 2507 @type repository_id: string 2508 @keyword ask: ask before making any change? 2509 @type ask: bool 2510 @return: execution status (0 = fine) 2511 @rtype: int 2512 """ 2513 self.output( 2514 "[%s] %s..." % ( 2515 purple(repository_id), darkgreen(_("initializing repository")), 2516 ), 2517 importance = 1, 2518 level = "info", header = darkgreen(" * ") 2519 ) 2520 self.close_repositories() 2521 2522 rc_question = self.ask_question( 2523 "[%s] %s" % ( 2524 purple(repository_id), 2525 teal(_("do you really want to initialize this repository ?")) 2526 ) 2527 ) 2528 if rc_question == _("No"): 2529 return 1 2530 2531 try: 2532 os.remove(self._get_local_repository_file(repository_id)) 2533 except OSError as err: 2534 if err.errno != errno.ENOENT: 2535 raise 2536 2537 # initialize 2538 dbconn = self.open_server_repository(repository_id, read_only = False, 2539 no_upload = True, is_new = True) 2540 dbconn.initializeRepository() 2541 dbconn.commit() 2542 2543 # create the store directory 2544 store_dir = self._get_local_store_directory(repository_id) 2545 if not os.path.isdir(store_dir): 2546 try: 2547 os.makedirs(store_dir) 2548 except (IOError, OSError) as err: 2549 self.output( 2550 "%s: %s" % (_("Cannot create store directory"), err), 2551 header=brown(" !!! "), 2552 importance=1, 2553 level="error") 2554 return 1 2555 2556 # create the upload directory 2557 upload_dir = self._get_local_upload_directory(repository_id) 2558 if not os.path.isdir(upload_dir): 2559 try: 2560 os.makedirs(upload_dir) 2561 except (IOError, OSError) as err: 2562 self.output( 2563 "%s: %s" % (_("Cannot create upload directory"), err), 2564 header=brown(" !!! "), 2565 importance=1, 2566 level="error") 2567 return 1 2568 2569 return 0
2570
2571 - def tag_packages(self, package_matches, package_tag, ask = True):
2572 """ 2573 Change version tag for given package matches. 2574 2575 @param package_matches: list of Entropy package matches 2576 @type package_matches: list 2577 @param package_tag: new Entropy package tag string 2578 @type package_tag: string 2579 @return: execution status (0 = fine) 2580 @rtype: int 2581 """ 2582 try: 2583 package_tag = str(package_tag) 2584 if " " in package_tag: 2585 raise ValueError 2586 except (UnicodeDecodeError, UnicodeEncodeError, ValueError,): 2587 self.output( 2588 "%s: %s" % ( 2589 blue(_("Invalid tag specified")), 2590 package_tag, 2591 ), 2592 importance = 1, level = "error", header = darkred(" !! ") 2593 ) 2594 return 1 2595 2596 pkg_map = {} 2597 for pkg_id, pkg_repo in package_matches: 2598 obj = pkg_map.setdefault(pkg_repo, []) 2599 obj.append(pkg_id) 2600 2601 for pkg_repo in sorted(pkg_map.keys()): 2602 switched = self._move_packages(pkg_map[pkg_repo], pkg_repo, 2603 pkg_repo, ask = ask, do_copy = True, new_tag = package_tag) 2604 if not switched: 2605 return 1 2606 return 0
2607
2608 - def flushback_packages(self, repository_id, from_branches, ask = True):
2609 """ 2610 When creating a new branch, for space reasons, packages are not 2611 moved to a new location. This works fine until old branch is removed. 2612 To avoid inconsistences, before deciding to do that, all the packages 2613 in the old branch should be flushed back to the the currently configured 2614 branch. 2615 2616 @param repository_id: repository identifier 2617 @type repository_id: string 2618 @param from_branches: list of branches to move packages from 2619 @type from_branches: list 2620 @keyword ask: ask before making any change? 2621 @type ask: bool 2622 @return execution status (0 = fine) 2623 @rtype: int 2624 """ 2625 branch = self._settings['repositories']['branch'] 2626 2627 if branch in from_branches: 2628 from_branches = [x for x in from_branches if x != branch] 2629 2630 self.output( 2631 "[%s=>%s|%s] %s" % ( 2632 darkgreen(', '.join(from_branches)), 2633 darkred(branch), 2634 brown(repository_id), 2635 blue(_("flushing back selected packages from branches")), 2636 ), 2637 importance = 2, 2638 level = "info", 2639 header = red(" @@ ") 2640 ) 2641 2642 dbconn = self.open_server_repository(repository_id, read_only = True, 2643 no_upload = True) 2644 2645 package_id_map = dict(((x, [],) for x in from_branches)) 2646 package_ids = dbconn.listAllPackageIds(order_by = 'atom') 2647 for package_id in package_ids: 2648 download_url = dbconn.retrieveDownloadURL(package_id) 2649 url_br = self._get_branch_from_download_relative_uri( 2650 download_url) 2651 if url_br in from_branches: 2652 package_id_map[url_br].append(package_id) 2653 2654 mapped_branches = [x for x in package_id_map if package_id_map[x]] 2655 if not mapped_branches: 2656 self.output( 2657 "[%s=>%s|%s] %s !" % ( 2658 darkgreen(', '.join(from_branches)), 2659 darkred(branch), 2660 brown(repository_id), 2661 blue(_("nothing to do")), 2662 ), 2663 importance = 0, 2664 level = "warning", 2665 header = blue(" @@ ") 2666 ) 2667 return 0 2668 2669 2670 all_fine = True 2671 tmp_down_dir = const_mkdtemp(prefix="entropy.server") 2672 2673 download_queue = {} 2674 dbconn = self.open_server_repository(repository_id, read_only = False, 2675 no_upload = True) 2676 2677 def generate_queue(branch, repository_id, from_branch, down_q, 2678 package_id_map): 2679 2680 self.output( 2681 "[%s=>%s|%s] %s" % ( 2682 darkgreen(from_branch), 2683 darkred(branch), 2684 brown(repository_id), 2685 brown(_("these are the packages that will be flushed")), 2686 ), 2687 importance = 1, 2688 level = "info", 2689 header = brown(" @@ ") 2690 ) 2691 2692 2693 for package_id in package_id_map[from_branch]: 2694 atom = dbconn.retrieveAtom(package_id) 2695 self.output( 2696 "[%s=>%s|%s] %s" % ( 2697 darkgreen(from_branch), 2698 darkred(branch), 2699 brown(repository_id), 2700 purple(atom), 2701 ), 2702 importance = 0, 2703 level = "info", 2704 header = blue(" # ") 2705 ) 2706 pkg_fp = os.path.basename( 2707 dbconn.retrieveDownloadURL(package_id)) 2708 pkg_fp = os.path.join(tmp_down_dir, pkg_fp) 2709 down_q.append((pkg_fp, package_id,))
2710 2711 2712 for from_branch in sorted(mapped_branches): 2713 2714 download_queue[from_branch] = [] 2715 all_fine = False 2716 generate_queue(branch, repository_id, from_branch, 2717 download_queue[from_branch], package_id_map) 2718 2719 if ask: 2720 rc_question = self.ask_question( 2721 _("Would you like to continue ?")) 2722 if rc_question == _("No"): 2723 continue 2724 2725 for uri in self.remote_packages_mirrors(repository_id): 2726 2727 crippled_uri = EntropyTransceiver.get_uri_name(uri) 2728 2729 queue_map = {} 2730 2731 for pkg_fp, package_id in download_queue[from_branch]: 2732 down_url = dbconn.retrieveDownloadURL(package_id) 2733 down_rel = self.complete_remote_package_relative_path( 2734 down_url, repository_id) 2735 down_rel_dir = os.path.dirname(down_rel) 2736 obj = queue_map.setdefault(down_rel_dir, []) 2737 obj.append(pkg_fp) 2738 2739 errors = False 2740 m_fine_uris = set() 2741 m_broken_uris = set() 2742 2743 for down_rel_dir, downloader_queue in queue_map.items(): 2744 2745 downloader = self.Mirrors.TransceiverServerHandler( 2746 self, 2747 [uri], 2748 downloader_queue, 2749 critical_files = downloader_queue, 2750 txc_basedir = down_rel_dir, 2751 local_basedir = tmp_down_dir, 2752 download = True, 2753 repo = repository_id 2754 ) 2755 xerrors, xm_fine_uris, xm_broken_uris = downloader.go() 2756 if xerrors: 2757 errors = True 2758 m_fine_uris.update(xm_fine_uris) 2759 m_broken_uris.update(xm_broken_uris) 2760 2761 if not errors: 2762 for downloaded_path, package_id in \ 2763 download_queue[from_branch]: 2764 2765 self.output( 2766 "[%s=>%s|%s|%s] %s: %s" % ( 2767 darkgreen(from_branch), 2768 darkred(branch), 2769 brown(repository_id), 2770 dbconn.retrieveAtom(package_id), 2771 blue(_("checking package hash")), 2772 darkgreen(os.path.basename(downloaded_path)), 2773 ), 2774 importance = 0, 2775 level = "info", 2776 header = brown(" "), 2777 back = True 2778 ) 2779 2780 md5hash = entropy.tools.md5sum(downloaded_path) 2781 db_md5hash = dbconn.retrieveDigest(package_id) 2782 if md5hash != db_md5hash: 2783 errors = True 2784 self.output( 2785 "[%s=>%s|%s|%s] %s: %s" % ( 2786 darkgreen(from_branch), 2787 darkred(branch), 2788 brown(repository_id), 2789 dbconn.retrieveAtom(package_id), 2790 blue(_("hash does not match for")), 2791 darkgreen(os.path.basename(downloaded_path)), 2792 ), 2793 importance = 0, 2794 level = "error", 2795 header = brown(" ") 2796 ) 2797 continue 2798 2799 if errors: 2800 reason = _("wrong md5") 2801 if m_broken_uris: 2802 my_broken_uris = [ 2803 (EntropyTransceiver.get_uri_name(x), y,) \ 2804 for x, y in m_broken_uris] 2805 reason = my_broken_uris[0][1] 2806 2807 self.output( 2808 "[%s=>%s|%s] %s, %s: %s" % ( 2809 darkgreen(from_branch), 2810 darkred(branch), 2811 brown(repository_id), 2812 blue(_("download errors")), 2813 blue(_("reason")), 2814 reason, 2815 ), 2816 importance = 1, 2817 level = "error", 2818 header = darkred(" !!! ") 2819 ) 2820 # continuing if possible 2821 continue 2822 2823 all_fine = True 2824 2825 self.output( 2826 "[%s=>%s|%s] %s: %s" % ( 2827 darkgreen(from_branch), 2828 darkred(branch), 2829 brown(repository_id), 2830 blue(_("download completed successfully")), 2831 darkgreen(crippled_uri), 2832 ), 2833 importance = 1, 2834 level = "info", 2835 header =