1
2 """
3
4 @author: Fabio Erculiani <[email protected]>
5 @contact: [email protected]
6 @copyright: Fabio Erculiani
7 @license: GPL-2
8
9 B{Entropy Package Manager Client EntropyRepository plugin code}.
10
11 """
12 import codecs
13 import errno
14 import os
15 import shutil
16 import subprocess
17 import sys
18 import threading
19 import time
20
21 from entropy.const import const_debug_write, const_setup_perms, etpConst, \
22 const_set_nice_level, const_setup_file, const_convert_to_unicode, \
23 const_debug_enabled, const_mkdtemp, const_mkstemp, const_file_readable, \
24 const_file_writable
25 from entropy.output import blue, darkred, red, darkgreen, purple, teal, brown, \
26 bold, TextInterface
27 from entropy.dump import dumpobj, loadobj
28 from entropy.cache import EntropyCacher
29 from entropy.db import EntropyRepository
30 from entropy.exceptions import RepositoryError, SystemDatabaseError, \
31 PermissionDenied
32 from entropy.security import Repository as RepositorySecurity
33 from entropy.misc import TimeScheduled, ParallelTask
34 from entropy.fetchers import UrlFetcher
35 from entropy.i18n import _
36 from entropy.db.skel import EntropyRepositoryPlugin, EntropyRepositoryBase
37 from entropy.db.exceptions import IntegrityError, OperationalError, Error, \
38 DatabaseError
39 from entropy.core.settings.base import SystemSettings
40 from entropy.services.client import WebService
41 from entropy.client.services.interfaces import RepositoryWebService, \
42 RepositoryWebServiceFactory
43
44 import entropy.dep
45 import entropy.tools
46
47 __all__ = ["CachedRepository", "ClientEntropyRepositoryPlugin",
48 "InstalledPackagesRepository", "AvailablePackagesRepository",
49 "GenericRepository"]
52
53 - def __init__(self, client_interface, metadata = None):
54 """
55 Entropy client-side repository EntropyRepository Plugin class.
56 This class will be instantiated and automatically added to
57 EntropyRepository instances generated by Entropy Client.
58
59 @param client_interface: Entropy Client interface instance
60 @type client_interface: entropy.client.interfaces.Client class
61 @param metadata: any dict form metadata map (key => value)
62 @type metadata: dict
63 """
64 EntropyRepositoryPlugin.__init__(self)
65 self._client = client_interface
66 if metadata is None:
67 self._metadata = {}
68 else:
69 self._metadata = metadata
70
73
76
78 const_debug_write(__name__,
79 "ClientEntropyRepositoryPlugin: calling add_plugin_hook => %s" % (
80 self,)
81 )
82
83 out_intf = self._metadata.get('output_interface')
84 if out_intf is not None:
85 entropy_repository_instance.output = out_intf.output
86 entropy_repository_instance.ask_question = out_intf.ask_question
87
88 return 0
89
91 """
92 This kind of repository cannot have close() called directly, without
93 a valid token passed. This is because the class object is cached somewhere
94 and calling close() would turn into a software bug.
95 """
97 """
98 Set a token that can be used to validate close() calls. Calling
99 close() on these repos is prohibited and considered a software bug.
100 Only Entropy Client should be able to close them.
101 """
102 self._close_token = token
103
104 - def close(self, safe=False, _token = None):
105 """
106 Reimplemented from EntropyRepository
107 """
108 close_token = getattr(self, "_close_token", None)
109 if close_token is not None:
110 if (_token is None) or (_token != close_token):
111 raise PermissionDenied(
112 "cannot close this repository directly. Software bug!")
113 return EntropyRepository.close(self, safe=safe)
114
117 """
118 This class represents the installed packages repository and is a direct
119 subclass of EntropyRepository.
120 """
121
122
123 NAME = "__system__"
124
136
137 - def handlePackage(self, pkg_data, revision = None,
138 formattedContent = False):
139 """
140 Reimplemented from EntropyRepository.
141 """
142 removelist = self.getPackagesToRemove(
143 pkg_data['name'], pkg_data['category'],
144 pkg_data['slot'], pkg_data['injected']
145 )
146 for r_package_id in removelist:
147 self.removePackage(r_package_id)
148 return self.addPackage(pkg_data, revision = revision,
149 formatted_content = formattedContent)
150
153
154 """
155 AvailablePackagesRepository update logic class.
156 The required logic for updating a repository is stored here.
157 """
158 WEBSERV_CACHE_ID = 'webserv_repo/segment_'
159
160 FETCH_ERRORS = (
161 UrlFetcher.GENERIC_FETCH_WARN,
162 UrlFetcher.TIMEOUT_FETCH_ERROR,
163 UrlFetcher.GENERIC_FETCH_ERROR)
164
165 - def __init__(self, entropy_client, repository_id, force, gpg):
166 self.__force = force
167 self.__big_sock_timeout = 20
168 self._repository_id = repository_id
169 self._cacher = EntropyCacher()
170 self._entropy = entropy_client
171 self._settings = SystemSettings()
172 self._gpg_feature = gpg
173 self._supported_apis = etpConst['supportedapis']
174 self._supported_download_items = (
175 "db", "dbck", "dblight", "ck", "cklight", "compck",
176 "lock", "dbdump", "dbdumplight", "dbdumplightck", "dbdumpck",
177 "meta_file", "meta_file_gpg", "notice_board"
178 )
179 self._developer_repo = \
180 self._settings['repositories']['developer_repo']
181 self._differential_update = \
182 self._settings['repositories']['differential_update']
183 if self._developer_repo:
184 const_debug_write(__name__,
185 "__init__: developer repo mode enabled")
186 self.__webservices = None
187 self.__webservice = None
188 self.__repo_eapi = None
189
190 avail_data = self._settings['repositories']['available']
191 if self._repository_id not in avail_data:
192 raise KeyError("Repository not available")
193
194 @property
196 if self.__repo_eapi is None:
197 self.__repo_eapi = self.__get_repo_eapi()
198 return self.__repo_eapi
199
200 @_repo_eapi.setter
202 self.__repo_eapi = value
203
204 @property
206 if self.__webservices is None:
207 if hasattr(self._entropy, "RepositoryWebServices"):
208 self.__webservices = self._entropy.RepositoryWebServices()
209 else:
210
211
212 self.__webservices = RepositoryWebServiceFactory(self._entropy)
213
214
215 return self.__webservices
216
217 @property
219 if self.__webservice is None:
220 self.__webservice = self._webservices.new(self._repository_id)
221 self.__webservice._set_timeout(self.__big_sock_timeout)
222 return self.__webservice
223
232
241
253
270
272
273 try:
274 remote_package_ids = webserv.get_package_ids()
275 except WebService.WebServiceException as err:
276 const_debug_write(__name__,
277 "__get_webserv_database_differences: error: %s" % (err,))
278 return None, None
279
280 added = [x for x in remote_package_ids if x not in package_ids]
281 removed = [x for x in package_ids if x not in remote_package_ids]
282
283 added.sort()
284 return added, removed
285
304
306
307
308 mytxt = "%s %s, %s %s" % (
309 red(_("Injecting downloaded dump")),
310 darkgreen(etpConst['etpdatabasedumplight']),
311 red(_("please wait")),
312 red("..."),
313 )
314 self._entropy.output(
315 mytxt,
316 importance = 0,
317 level = "info",
318 header = "\t"
319 )
320 dbconn = self._entropy.open_generic_repository(dbfile,
321 xcache = False, indexing_override = False)
322 rc = dbconn.importRepository(dumpfile, dbfile)
323 dbconn.close()
324 return rc
325
327
328 eapi_env = os.getenv("FORCE_EAPI")
329 sqlite3_access = os.path.lexists("/usr/bin/sqlite3")
330 sqlite3_rc = subprocess.call("/usr/bin/sqlite3 -version > /dev/null",
331 shell = True)
332 try:
333 eapi_env_clear = int(eapi_env)
334 if eapi_env_clear not in self._supported_apis:
335 raise ValueError()
336 except (ValueError, TypeError,):
337 eapi_env_clear = None
338
339 repo_eapi = 2
340 eapi_avail = self.__check_webserv_availability()
341 if eapi_avail:
342 repo_eapi = 3
343 else:
344 if not sqlite3_access or entropy.tools.islive():
345 repo_eapi = 1
346 elif sqlite3_rc != 0:
347 repo_eapi = 1
348
349
350
351 if (eapi_env_clear is None) and (not self._differential_update) and \
352 (repo_eapi == 3):
353 const_debug_write(__name__,
354 "__get_repo_eapi: differential update is disabled !")
355 repo_eapi -= 1
356
357
358 if eapi_env_clear is not None:
359 repo_eapi = eapi_env_clear
360
361
362
363 self._developer_repo = False
364 const_debug_write(__name__,
365 "__get_repo_eapi: developer repo mode disabled FORCE_EAPI")
366
367 elif repo_eapi > 1 and self._developer_repo:
368
369 repo_eapi = 1
370
371 const_debug_write(__name__,
372 "__get_repo_eapi: final eapi set to %s" % (repo_eapi,))
373
374 return repo_eapi
375
377 """
378 Given a remote repository revision, return whether
379 the repository is updatable or not.
380 """
381 if self.__force:
382 return True
383
384 local = AvailablePackagesRepository.revision(
385 self._repository_id)
386 return local != revision
387
389 """
390 Verify that the database mirror URL is available and working.
391 If this is not the case, fallback to the first available
392 and working packages mirror URL.
393 """
394 repos_data = self._settings['repositories']
395 avail_data = repos_data['available']
396 repo_data = avail_data[self._repository_id]
397 database_uris = repo_data['databases']
398 basic_user = repo_data.get('username')
399 basic_pwd = repo_data.get('password')
400 https_validate_cert = not repo_data.get('https_validate_cert') == "false"
401
402 ws_revision = self._remote_webservice_revision()
403
404
405 revision = None
406 uri = None
407 cformat = None
408 for counter, uri_meta in enumerate(database_uris, 1):
409
410 mytxt = "%s: %s [%s]" % (
411 red(_("Scanning URL")),
412 teal(uri_meta['uri']),
413 brown(uri_meta['dbcformat']),)
414 self._entropy.output(
415 mytxt,
416 importance = 1,
417 level = "info",
418 header = blue(" # "),
419 back = True,
420 count = (counter, len(database_uris))
421 )
422
423 repo_uri = uri_meta['uri']
424
425 uri_revision = self._remote_revision(repo_uri,
426 http_basic_user = basic_user,
427 http_basic_pwd = basic_pwd,
428 https_validate_cert = https_validate_cert)
429
430 if uri_revision != -1:
431
432 mytxt = "%s: %s [%s]" % (
433 darkgreen(_("Selected URL")),
434 teal(uri_meta['uri']),
435 brown(uri_meta['dbcformat']),)
436 self._entropy.output(
437 mytxt,
438 importance = 1,
439 level = "info",
440 header = blue(" # ")
441 )
442
443 revision = uri_revision
444 uri = repo_uri
445 cformat = uri_meta['dbcformat']
446 break
447
448 if uri is not None:
449 if ws_revision is not None:
450 return ws_revision, uri, cformat
451 elif revision is not None:
452 return revision, uri, cformat
453
454 self._entropy.output(
455 "%s: %s" % (
456 darkred(_("Attention")),
457 brown(_("repository is not available at the database URLs")),
458 ),
459 importance = 1, level = "warning", header = "\t",
460 )
461 self._entropy.output(
462 "%s" % (
463 purple(
464 _("Looking for an alternate route using package mirrors")
465 ),),
466 importance = 1, level = "warning", header = "\t",
467 )
468
469 package_uris = repo_data['plain_packages']
470 default_cformat = etpConst['etpdatabasefileformat']
471 for package_uri in package_uris:
472 self._entropy.output(
473 "%s:" % (brown(_("Checking repository URL")),),
474 importance = 1, level = "warning", header = "\t",
475 )
476 self._entropy.output(
477 "%s" % (package_uri,),
478 importance = 1, level = "warning", header = "\t",
479 )
480
481 url = entropy.tools.expand_plain_database_mirror(
482 package_uri, repos_data['product'],
483 self._repository_id,
484 repos_data['branch'])
485 if url == uri:
486
487 continue
488
489 revision = self._remote_revision(url,
490 http_basic_user = basic_user,
491 http_basic_pwd = basic_pwd,
492 https_validate_cert = https_validate_cert)
493 if revision != -1:
494
495 self._entropy.output(
496 "%s:" % (brown(_("Found repository at URL")),),
497 importance = 1, level = "warning", header = "\t",
498 )
499 self._entropy.output(
500 "%s" % (url,),
501 importance = 1, level = "warning", header = "\t",
502 )
503 return revision, url, default_cformat
504
505 self._entropy.output(
506 "%s" % (
507 purple(
508 _("Unable to find alternate repository mirrors. Sorry.")
509 ),),
510 importance = 1, level = "warning", header = "\t",
511 )
512 return None
513
556
559
561 """
562 Make sure that the local repository directory has valid
563 permissions and ownership.
564 """
565 avail_data = self._settings['repositories']['available']
566 repo_data = avail_data[self._repository_id]
567
568
569 if not os.path.isdir(repo_data['dbpath']):
570 os.makedirs(repo_data['dbpath'], 0o755)
571
572 try:
573 items = os.listdir(etpConst['etpdatabaseclientdir'])
574 except OSError:
575 items = []
576
577
578
579 for item in items:
580 repo_dir_path = os.path.join(
581 etpConst['etpdatabaseclientdir'],
582 item)
583 if not os.path.isdir(repo_dir_path):
584 continue
585 const_setup_perms(
586 repo_dir_path,
587 etpConst['entropygid'],
588 f_perms = 0o644)
589
591 sys_set = self._settings
592 avail_data = sys_set['repositories']['available']
593 repo_dbpath = avail_data[self._repository_id]['dbpath']
594 shutil.rmtree(repo_dbpath, True)
595
597
598 mytxt = "%s ..." % (red(_("Downloading repository")),)
599 self._entropy.output(
600 mytxt,
601 importance = 1,
602 level = "info",
603 header = "\t"
604 )
605
606 downloaded_item = None
607 down_status = False
608 sig_status = False
609 if self._repo_eapi == 2:
610
611 down_item = "dbdumplight"
612
613 down_status = self._download_item(
614 uri, down_item, cmethod,
615 disallow_redirect = True)
616 if down_status:
617
618 sig_status = self._download_item(
619 uri, down_item, cmethod,
620 disallow_redirect = True,
621 get_signature = True)
622
623 downloaded_item = down_item
624
625 if not down_status:
626
627 self._repo_eapi = 1
628 down_item = "dblight"
629 if self._developer_repo:
630
631 down_item = "db"
632 const_debug_write(__name__,
633 "__handle_database_download: developer repo mode enabled")
634
635 down_status = self._download_item(
636 uri, down_item, cmethod,
637 disallow_redirect = True)
638 if down_status:
639 sig_status = self._download_item(
640 uri, down_item, cmethod,
641 disallow_redirect = True,
642 get_signature = True)
643
644 downloaded_item = down_item
645
646 if not down_status:
647 mytxt = "%s: %s." % (bold(_("Attention")),
648 red(_("unable to download the repository")),)
649 self._entropy.output(
650 mytxt,
651 importance = 1,
652 level = "warning",
653 header = "\t"
654 )
655
656 return down_status, sig_status, downloaded_item
657
659
660 downitem = 'cklight'
661 if self._developer_repo:
662 downitem = 'dbck'
663 if self._repo_eapi == 2:
664 downitem = 'dbdumplightck'
665
666 garbage_url, hashfile = self._construct_paths(
667 uri, downitem, cmethod)
668 mytxt = "%s %s %s" % (
669 red(_("Downloading checksum")),
670 darkgreen(os.path.basename(hashfile)),
671 red("..."),
672 )
673
674 self._entropy.output(
675 mytxt,
676 importance = 0,
677 level = "info",
678 header = "\t"
679 )
680
681 db_down_status = self._download_item(
682 uri, downitem, cmethod,
683 disallow_redirect = True)
684
685 if not db_down_status and (downitem not in ('cklight', 'dbck',)):
686
687 retryitem = 'cklight'
688 if self._developer_repo:
689 retryitem = 'dbck'
690 db_down_status = self._download_item(
691 uri, retryitem, cmethod,
692 disallow_redirect = True)
693
694 if not db_down_status:
695 mytxt = "%s %s !" % (
696 red(_("Cannot fetch checksum")),
697 red(_("Cannot verify repository integrity")),
698 )
699 self._entropy.output(
700 mytxt,
701 importance = 1,
702 level = "warning",
703 header = "\t"
704 )
705 return db_down_status
706
708 enc = etpConst['conf_encoding']
709 with codecs.open(md5_checksum_path, "r", encoding=enc) as ck_f:
710 md5hash = ck_f.readline().strip()
711 if not md5hash:
712 return False
713 md5hash = md5hash.split()[0]
714 return entropy.tools.compare_md5(file_path, md5hash)
715
717
718 sys_settings_repos = self._settings['repositories']
719 avail_config = sys_settings_repos['available'][self._repository_id]
720
721 sep = os.path.sep
722 if self._repo_eapi == 1:
723 if self._developer_repo:
724 remote_gb, dbfile = self._construct_paths(
725 uri, "db", cmethod)
726 remote_gb, md5file = self._construct_paths(
727 uri, "dbck", cmethod)
728 else:
729 remote_gb, dbfile = self._construct_paths(
730 uri, "dblight", cmethod)
731 remote_gb, md5file = self._construct_paths(
732 uri, "cklight", cmethod)
733
734 elif self._repo_eapi == 2:
735 remote_gb, dbfile = self._construct_paths(
736 uri, "dbdumplight", cmethod)
737 remote_gb, md5file = self._construct_paths(
738 uri, "dbdumplightck", cmethod)
739
740 else:
741 raise AttributeError("EAPI must be = 1 or 2")
742
743 try:
744 return self.__verify_file_checksum(dbfile, md5file)
745 except (OSError, IOError) as err:
746 if err.errno != errno.ENOENT:
747 raise
748 return -1
749
751
752 rc = 0
753 path = None
754 sys_set_repos = self._settings['repositories']['available']
755 repo_data = sys_set_repos[self._repository_id]
756
757 garbage, myfile = self._construct_paths(uri, down_item, cmethod)
758
759 if self._repo_eapi in (1, 2,):
760 try:
761
762 myfunc = getattr(entropy.tools, cmethod[1])
763 path = myfunc(myfile)
764
765 if self._repo_eapi == 1:
766 new_path = os.path.join(os.path.dirname(path),
767 etpConst['etpdatabasefile'])
768 os.rename(path, new_path)
769 path = new_path
770
771 except (OSError, EOFError):
772 rc = 1
773
774 else:
775 mytxt = "invalid EAPI must be = 1 or 2"
776 raise AttributeError(mytxt)
777
778 if rc == 0:
779 const_setup_file(path, etpConst['entropygid'], 0o644,
780 uid = etpConst['uid'])
781
782 return rc
783
785 """
786 Unpack the downloaded database.
787 """
788 file_to_unpack = etpConst['etpdatabasedump']
789 if self._repo_eapi == 1:
790 file_to_unpack = etpConst['etpdatabasefile']
791 elif self._repo_eapi == 2:
792 file_to_unpack = etpConst['etpdatabasedumplight']
793
794 mytxt = "%s %s %s" % (red(_("Unpacking database to")),
795 darkgreen(file_to_unpack), red("..."),)
796 self._entropy.output(
797 mytxt,
798 importance = 0,
799 level = "info",
800 header = "\t"
801 )
802
803 myitem = 'dblight'
804 if self._repo_eapi == 2:
805 myitem = 'dbdumplight'
806 elif self._developer_repo:
807 myitem = 'db'
808
809 myrc = self.__unpack_downloaded_database(uri, myitem, cmethod)
810 if myrc != 0:
811 mytxt = "%s %s !" % (red(_("Cannot unpack compressed package")),
812 red(_("Skipping repository")),)
813 self._entropy.output(
814 mytxt,
815 importance = 1,
816 level = "warning",
817 header = "\t"
818 )
819 return False, myitem
820 return True, myitem
821
823 cur_rev = AvailablePackagesRepository.revision(self._repository_id)
824 repo_data = self._settings['repositories']
825 db_data = repo_data['available'][self._repository_id]
826 db_data['dbrevision'] = "0"
827 if cur_rev != -1:
828 db_data['dbrevision'] = str(cur_rev)
829
830 rev_file = os.path.join(db_data['dbpath'],
831 etpConst['etpdatabaserevisionfile'])
832 enc = etpConst['conf_encoding']
833 with codecs.open(rev_file, "w", encoding=enc) as rev_f:
834
835 rev_f.write("%s\n" % (revision,))
836 rev_f.flush()
837
839 def tell_error(err):
840 mytxt = "%s: %s" % (darkred(_("Repository is invalid")),
841 repr(err),)
842 self._entropy.output(
843 mytxt,
844 importance = 1,
845 level = "error",
846 header = "\t"
847 )
848 try:
849 dbconn = self._entropy.open_repository(self._repository_id)
850 except RepositoryError as err:
851 tell_error(err)
852 return False
853 try:
854 dbconn.validate()
855 except SystemDatabaseError as err:
856 tell_error(err)
857 return False
858 return True
859
883
885 """
886 Build a remote URL and a local path for a supported resource item.
887 """
888 if item not in self._supported_download_items:
889 raise AttributeError("Invalid item: %s" % (item,))
890
891 items_needing_cmethod = (
892 "db", "dbck", "dblight", "cklight", "dbdump", "dbdumpck",
893 "dbdumplight", "dbdumplightck", "compck",
894 )
895 if (item in items_needing_cmethod) and (cmethod is None):
896 mytxt = "For %s, cmethod can't be None" % (item,)
897 raise AttributeError(mytxt)
898
899 avail_data = self._settings['repositories']['available']
900 repo_data = avail_data[self._repository_id]
901
902 repo_dbpath = repo_data['dbpath']
903 ec_hash = etpConst['etpdatabasehashfile']
904 repo_lock_file = etpConst['etpdatabasedownloadlockfile']
905 notice_board_filename = os.path.basename(repo_data['notice_board'])
906 meta_file = etpConst['etpdatabasemetafilesfile']
907 meta_file_gpg = etpConst['etpdatabasemetafilesfile'] + \
908 etpConst['etpgpgextension']
909 md5_ext = etpConst['packagesmd5fileext']
910 ec_cm2 = None
911 ec_cm3 = None
912 ec_cm4 = None
913 ec_cm5 = None
914 ec_cm6 = None
915 ec_cm7 = None
916 ec_cm8 = None
917 ec_cm9 = None
918 if cmethod is not None:
919 ec_cm2 = etpConst[cmethod[2]]
920 ec_cm3 = etpConst[cmethod[3]]
921 ec_cm4 = etpConst[cmethod[4]]
922 ec_cm5 = etpConst[cmethod[5]]
923 ec_cm6 = etpConst[cmethod[6]]
924 ec_cm7 = etpConst[cmethod[7]]
925 ec_cm8 = etpConst[cmethod[8]]
926 ec_cm9 = etpConst[cmethod[9]]
927
928 mymap = {
929 'db': (
930 "%s/%s" % (uri, ec_cm2,),
931 "%s/%s" % (repo_dbpath, ec_cm2,),
932 ),
933 'dbck': (
934 "%s/%s" % (uri, ec_cm9,),
935 "%s/%s" % (repo_dbpath, ec_cm9,),
936 ),
937 'dblight': (
938 "%s/%s" % (uri, ec_cm7,),
939 "%s/%s" % (repo_dbpath, ec_cm7,),
940 ),
941 'dbdump': (
942 "%s/%s" % (uri, ec_cm3,),
943 "%s/%s" % (repo_dbpath, ec_cm3,),
944 ),
945 'dbdumplight': (
946 "%s/%s" % (uri, ec_cm5,),
947 "%s/%s" % (repo_dbpath, ec_cm5,),
948 ),
949 'ck': (
950 "%s/%s" % (uri, ec_hash,),
951 "%s/%s" % (repo_dbpath, ec_hash,),
952 ),
953 'cklight': (
954 "%s/%s" % (uri, ec_cm8,),
955 "%s/%s" % (repo_dbpath, ec_cm8,),
956 ),
957 'compck': (
958 "%s/%s%s" % (uri, ec_cm2, md5_ext,),
959 "%s/%s%s" % (repo_dbpath, ec_cm2, md5_ext,),
960 ),
961 'dbdumpck': (
962 "%s/%s" % (uri, ec_cm4,),
963 "%s/%s" % (repo_dbpath, ec_cm4,),
964 ),
965 'dbdumplightck': (
966 "%s/%s" % (uri, ec_cm6,),
967 "%s/%s" % (repo_dbpath, ec_cm6,),
968 ),
969 'lock': (
970 "%s/%s" % (uri, repo_lock_file,),
971 "%s/%s" % (repo_dbpath, repo_lock_file,),
972 ),
973 'notice_board': (
974 repo_data['notice_board'],
975 "%s/%s" % (repo_dbpath, notice_board_filename,),
976 ),
977 'meta_file': (
978 "%s/%s" % (uri, meta_file,),
979 "%s/%s" % (repo_dbpath, meta_file,),
980 ),
981 'meta_file_gpg': (
982 "%s/%s" % (uri, meta_file_gpg,),
983 "%s/%s" % (repo_dbpath, meta_file_gpg,),
984 ),
985 }
986
987 url, path = mymap.get(item)
988 if get_signature:
989 url = self.__append_gpg_signature_to_path(url)
990 path = self.__append_gpg_signature_to_path(path)
991
992 return url, path
993
994 - def _download_item(self, uri, item, cmethod = None,
995 disallow_redirect = True, get_signature = False):
996
997 my_repos = self._settings['repositories']
998 avail_data = my_repos['available']
999 repo_data = avail_data[self._repository_id]
1000
1001 basic_user = repo_data.get('username')
1002 basic_pwd = repo_data.get('password')
1003 https_validate_cert = not repo_data.get('https_validate_cert') == "false"
1004
1005 url, filepath = self._construct_paths(
1006 uri, item, cmethod, get_signature = get_signature)
1007
1008
1009
1010
1011 temp_filepath = filepath + ".edownload"
1012
1013
1014
1015
1016 if os.path.isfile(temp_filepath):
1017 os.remove(temp_filepath)
1018 filepath_dir = os.path.dirname(temp_filepath)
1019
1020 if not os.path.isdir(filepath_dir) and not \
1021 os.path.lexists(filepath_dir):
1022
1023 os.makedirs(filepath_dir, 0o755)
1024 const_setup_perms(filepath_dir, etpConst['entropygid'],
1025 f_perms = 0o644)
1026
1027 try:
1028
1029 fetcher = self._entropy._url_fetcher(
1030 url,
1031 temp_filepath,
1032 resume = False,
1033 disallow_redirect = disallow_redirect,
1034 http_basic_user = basic_user,
1035 http_basic_pwd = basic_pwd,
1036 https_validate_cert = https_validate_cert
1037 )
1038
1039 rc = fetcher.download()
1040 if rc in self.FETCH_ERRORS:
1041 return False
1042 try:
1043 os.rename(temp_filepath, filepath)
1044 except (OSError, IOError) as err:
1045 if err.errno != errno.ENOENT:
1046 raise
1047 return False
1048
1049 const_setup_file(filepath, etpConst['entropygid'], 0o644,
1050 uid = etpConst['uid'])
1051 return True
1052
1053 finally:
1054
1055 try:
1056 os.remove(temp_filepath)
1057 except (OSError, IOError) as err:
1058 if err.errno != errno.ENOENT:
1059 raise
1060
1062 """
1063 Returns whether the repository is remotely locked or not.
1064
1065 @return: repository being remotely locked
1066 @rtype:bool
1067 """
1068 rc = self._download_item(uri, "lock", disallow_redirect = True)
1069 if rc:
1070 return False
1071 return True
1072
1074 """
1075 Download a set of "standard" files from the repository mirror.
1076 """
1077 repos_data = self._settings['repositories']
1078 repo_data = repos_data['available'][self._repository_id]
1079 notice_board = os.path.basename(repo_data['local_notice_board'])
1080 db_meta_file = etpConst['etpdatabasemetafilesfile']
1081 db_meta_file_gpg = etpConst['etpdatabasemetafilesfile'] + \
1082 etpConst['etpgpgextension']
1083
1084 objects_to_unpack = ("meta_file",)
1085
1086 download_items = [
1087 (
1088 "meta_file",
1089 db_meta_file,
1090 False,
1091 "%s %s %s" % (
1092 red(_("Downloading repository metafile")),
1093 darkgreen(db_meta_file),
1094 red("..."),
1095 )
1096 ),
1097 (
1098 "meta_file_gpg",
1099 db_meta_file_gpg,
1100 True,
1101 "%s %s %s" % (
1102 red(_("Downloading GPG signature of repository metafile")),
1103 darkgreen(db_meta_file_gpg),
1104 red("..."),
1105 )
1106 ),
1107 (
1108 "notice_board",
1109 notice_board,
1110 True,
1111 "%s %s %s" % (
1112 red(_("Downloading Notice Board")),
1113 darkgreen(notice_board),
1114 red("..."),
1115 )
1116 ),
1117 ]
1118
1119 def my_show_info(txt):
1120 self._entropy.output(
1121 txt,
1122 importance = 0,
1123 level = "info",
1124 header = "\t",
1125 back = True
1126 )
1127
1128 def my_show_down_status(message, mytype):
1129 self._entropy.output(
1130 message,
1131 importance = 0,
1132 level = mytype,
1133 header = "\t"
1134 )
1135
1136 def my_show_file_unpack(fp):
1137 self._entropy.output(
1138 "%s: %s" % (darkgreen(_("unpacked meta file")), brown(fp),),
1139 header = blue("\t << ")
1140 )
1141
1142 def my_show_file_rm(fp):
1143 self._entropy.output(
1144 "%s: %s" % (darkgreen(_("removed meta file")), purple(fp),),
1145 header = blue("\t << ")
1146 )
1147
1148 downloaded_files = []
1149
1150 for item, myfile, ignorable, mytxt in download_items:
1151
1152 my_show_info(mytxt)
1153 mystatus = self._download_item(uri, item, disallow_redirect = True)
1154 mytype = 'info'
1155 myurl, mypath = self._construct_paths(uri, item, None)
1156
1157
1158 if not mystatus:
1159 if ignorable:
1160 message = "%s: %s." % (blue(myfile),
1161 red(_("not available, it's ok")))
1162 else:
1163 mytype = 'warning'
1164 message = "%s: %s." % (blue(myfile),
1165 darkred(_("not available, not very ok!")))
1166 my_show_down_status(message, mytype)
1167
1168
1169 if os.path.isfile(mypath):
1170 try:
1171 os.remove(mypath)
1172 except OSError:
1173 continue
1174
1175 continue
1176
1177 message = "%s: %s." % (blue(myfile),
1178 darkgreen(_("available, w00t!")))
1179 my_show_down_status(message, mytype)
1180 downloaded_files.append(mypath)
1181
1182 if item not in objects_to_unpack:
1183 continue
1184 if not const_file_readable(mypath):
1185 continue
1186
1187 tmpdir = const_mkdtemp(prefix="_standard_items_download")
1188 repo_dir = repo_data['dbpath']
1189 enc = etpConst['conf_encoding']
1190 try:
1191 done = entropy.tools.universal_uncompress(mypath, tmpdir,
1192 catch_empty = True)
1193 if not done:
1194 mytype = 'warning'
1195 message = "%s: %s." % (blue(myfile),
1196 darkred(_("cannot be unpacked, not very ok!")))
1197 my_show_down_status(message, mytype)
1198 continue
1199 myfiles_to_move = set(os.listdir(tmpdir))
1200
1201
1202 files_not_found_file = etpConst['etpdatabasemetafilesnotfound']
1203 if files_not_found_file in myfiles_to_move:
1204 myfiles_to_move.remove(files_not_found_file)
1205 fnf_path = os.path.join(tmpdir, files_not_found_file)
1206
1207 f_nf = []
1208 try:
1209 with codecs.open(fnf_path, "r", encoding=enc) as f:
1210 f_nf += [x.strip() for x in f.readlines()]
1211 except (OSError, IOError) as err:
1212 if err.errno != errno.ENOENT:
1213 raise
1214 else:
1215 for myfile in f_nf:
1216 myfile = os.path.basename(myfile)
1217 myfpath = os.path.join(repo_dir, myfile)
1218 try:
1219 os.remove(myfpath)
1220 my_show_file_rm(myfile)
1221 except (OSError, IOError) as err:
1222 if err.errno != errno.ENOENT:
1223 raise
1224
1225 for myfile in sorted(myfiles_to_move):
1226 from_mypath = os.path.join(tmpdir, myfile)
1227 to_mypath = os.path.join(repo_dir, myfile)
1228 try:
1229 os.rename(from_mypath, to_mypath)
1230 my_show_file_unpack(myfile)
1231 except OSError:
1232
1233 try:
1234 shutil.copy2(from_mypath, to_mypath)
1235 my_show_file_unpack(myfile)
1236 os.remove(from_mypath)
1237 except (shutil.Error, IOError, OSError,):
1238 continue
1239 continue
1240
1241 const_setup_file(to_mypath, etpConst['entropygid'], 0o644,
1242 uid = etpConst['uid'])
1243
1244 finally:
1245 shutil.rmtree(tmpdir, True)
1246
1247 repo_r = AvailablePackagesRepository.revision(self._repository_id)
1248 mytxt = "%s: %s" % (
1249 red(_("Repository revision")),
1250 bold(str(repo_r)),
1251 )
1252 self._entropy.output(
1253 mytxt,
1254 importance = 1,
1255 level = "info",
1256 header = "\t"
1257 )
1258
1259 return downloaded_files
1260
1262
1263 dbitem = "dblight"
1264 if self._repo_eapi == 2:
1265 dbitem = "dbdumplight"
1266 elif self._developer_repo:
1267 dbitem = "db"
1268 garbage, dbfilename = self._construct_paths(uri, dbitem, cmethod)
1269
1270
1271 mytxt = "%s %s %s" % (
1272 red(_("Checking downloaded repository")),
1273 darkgreen(os.path.basename(dbfilename)),
1274 red("..."),
1275 )
1276 self._entropy.output(
1277 mytxt,
1278 importance = 0,
1279 back = True,
1280 level = "info",
1281 header = "\t"
1282 )
1283 db_status = self.__verify_database_checksum(uri, cmethod)
1284 if db_status == -1:
1285 mytxt = "%s. %s !" % (
1286 red(_("Cannot open digest")),
1287 red(_("Cannot verify repository integrity")),
1288 )
1289 self._entropy.output(
1290 mytxt,
1291 importance = 1,
1292 level = "warning",
1293 header = "\t"
1294 )
1295 elif db_status:
1296 mytxt = "%s: %s" % (
1297 red(_("Downloaded repository status")),
1298 bold(_("OK")),
1299 )
1300 self._entropy.output(
1301 mytxt,
1302 importance = 1,
1303 level = "info",
1304 header = "\t"
1305 )
1306 else:
1307 mytxt = "%s: %s" % (
1308 red(_("Downloaded repository status")),
1309 darkred(_("ERROR")),
1310 )
1311 self._entropy.output(
1312 mytxt,
1313 importance = 1,
1314 level = "error",
1315 header = "\t"
1316 )
1317 mytxt = "%s. %s" % (
1318 red(_("An error occurred while checking repository integrity")),
1319 red(_("Giving up")),
1320 )
1321 self._entropy.output(
1322 mytxt,
1323 importance = 1,
1324 level = "error",
1325 header = "\t"
1326 )
1327 return 1
1328 return 0
1329
1331
1332 my_repos = self._settings['repositories']
1333 avail_data = my_repos['available']
1334 repo_data = avail_data[self._repository_id]
1335 gpg_path = repo_data['gpg_pubkey']
1336
1337 if not const_file_readable(gpg_path):
1338 return False
1339
1340 def do_warn_user(fingerprint):
1341 mytxt = purple(_("Make sure to verify the imported key and "
1342 "set an appropriate trust level"))
1343 self._entropy.output(
1344 mytxt + ":",
1345 level = "warning",
1346 header = "\t"
1347 )
1348 mytxt = brown("gpg --homedir '%s' --edit-key '%s'" % (
1349 etpConst['etpclientgpgdir'], fingerprint,)
1350 )
1351 self._entropy.output(
1352 "$ " + mytxt,
1353 level = "warning",
1354 header = "\t"
1355 )
1356
1357 try:
1358 repo_sec = self._entropy.RepositorySecurity()
1359 except RepositorySecurity.GPGError:
1360 mytxt = "%s," % (
1361 purple(_("This repository suports GPG-signed packages")),
1362 )
1363 self._entropy.output(
1364 mytxt,
1365 level = "warning",
1366 header = "\t"
1367 )
1368 mytxt = purple(_("you may want to install GnuPG to take "
1369 "advantage of this feature"))
1370 self._entropy.output(
1371 mytxt,
1372 level = "warning",
1373 header = "\t"
1374 )
1375 return False
1376
1377 pk_expired = False
1378 try:
1379 pk_avail = repo_sec.is_pubkey_available(self._repository_id)
1380 except repo_sec.KeyExpired:
1381 pk_avail = False
1382 pk_expired = True
1383
1384 if pk_avail:
1385
1386 tmp_dir = const_mkdtemp(prefix="_install_gpg_key")
1387 repo_tmp_sec = self._entropy.RepositorySecurity(
1388 keystore_dir = tmp_dir)
1389
1390 try:
1391 downloaded_key_fp = repo_tmp_sec.install_key(
1392 self._repository_id, gpg_path)
1393 except RepositorySecurity.GPGError:
1394 downloaded_key_fp = None
1395
1396 fingerprint = repo_sec.get_key_metadata(
1397 self._repository_id)['fingerprint']
1398 shutil.rmtree(tmp_dir, True)
1399
1400 if downloaded_key_fp != fingerprint and \
1401 (downloaded_key_fp is not None):
1402 mytxt = "%s: %s !!!" % (
1403 purple(_("GPG key changed for")),
1404 bold(self._repository_id),
1405 )
1406 self._entropy.output(
1407 mytxt,
1408 level = "warning",
1409 header = "\t"
1410 )
1411 mytxt = "[%s => %s]" % (
1412 darkgreen(fingerprint),
1413 purple(downloaded_key_fp),
1414 )
1415 self._entropy.output(
1416 mytxt,
1417 level = "warning",
1418 header = "\t"
1419 )
1420 do_warn_user(downloaded_key_fp)
1421 else:
1422 mytxt = "%s: %s" % (
1423 purple(_("GPG key already installed for")),
1424 bold(self._repository_id),
1425 )
1426 self._entropy.output(
1427 mytxt,
1428 level = "info",
1429 header = "\t"
1430 )
1431 do_warn_user(fingerprint)
1432 return True
1433
1434 elif pk_expired:
1435 mytxt = "%s: %s" % (
1436 purple(_("GPG key EXPIRED for repository")),
1437 bold(self._repository_id),
1438 )
1439 self._entropy.output(
1440 mytxt,
1441 level = "warning",
1442 header = "\t"
1443 )
1444
1445
1446
1447 mytxt = "%s: %s" % (
1448 purple(_("Installing GPG key for repository")),
1449 brown(self._repository_id),
1450 )
1451 self._entropy.output(
1452 mytxt,
1453 level = "info",
1454 header = "\t",
1455 back = True
1456 )
1457
1458 try_ignore = False
1459 while True:
1460 try:
1461 fingerprint = repo_sec.install_key(self._repository_id,
1462 gpg_path)
1463 except RepositorySecurity.NothingImported as err:
1464 if try_ignore:
1465 mytxt = "%s: %s" % (
1466 darkred(_("Error during GPG key installation")),
1467 err,
1468 )
1469 self._entropy.output(
1470 mytxt,
1471 level = "error",
1472 header = "\t"
1473 )
1474 return False
1475 self._entropy.output(
1476 purple(_("GPG key seems already installed but "
1477 "not properly recorded, resetting")),
1478 level = "warning",
1479 header = "\t"
1480 )
1481 target_fingerprint = repo_sec.get_key_fingerprint(gpg_path)
1482 if target_fingerprint is not None:
1483
1484
1485 dead_repository_ids = set()
1486 for _repository_id, key_meta in repo_sec.get_keys().items():
1487 if key_meta['fingerprint'] == target_fingerprint:
1488 dead_repository_ids.add(_repository_id)
1489 for _repository_id in dead_repository_ids:
1490 try:
1491 repo_sec.delete_pubkey(_repository_id)
1492 except KeyError:
1493
1494 pass
1495 try_ignore = True
1496 continue
1497 except RepositorySecurity.GPGError as err:
1498 mytxt = "%s: %s" % (
1499 darkred(_("Error during GPG key installation")),
1500 err,
1501 )
1502 self._entropy.output(
1503 mytxt,
1504 level = "error",
1505 header = "\t"
1506 )
1507 return False
1508 break
1509
1510 mytxt = "%s: %s" % (
1511 purple(_("Successfully installed GPG key for repository")),
1512 brown(self._repository_id),
1513 )
1514 self._entropy.output(
1515 mytxt,
1516 level = "info",
1517 header = "\t"
1518 )
1519 mytxt = "%s: %s" % (
1520 darkgreen(_("Fingerprint")),
1521 bold(fingerprint),
1522 )
1523 self._entropy.output(
1524 mytxt,
1525 level = "info",
1526 header = "\t"
1527 )
1528 do_warn_user(fingerprint)
1529 return True
1530
1532
1533 try:
1534 repo_sec = self._entropy.RepositorySecurity()
1535 except RepositorySecurity.GPGServiceNotAvailable:
1536
1537 return 0
1538
1539 gpg_sign_ext = etpConst['etpgpgextension']
1540 sign_files = [x for x in downloaded_files if \
1541 x.endswith(gpg_sign_ext)]
1542 sign_files = [x for x in sign_files if const_file_readable(x)]
1543
1544 to_be_verified = []
1545
1546 for sign_path in sign_files:
1547 target_path = sign_path[:-len(gpg_sign_ext)]
1548 if const_file_readable(target_path):
1549 to_be_verified.append((target_path, sign_path,))
1550
1551 gpg_rc = 0
1552
1553 for target_path, sign_path in to_be_verified:
1554
1555 file_name = os.path.basename(target_path)
1556
1557 mytxt = "%s: %s ..." % (
1558 darkgreen(_("Verifying GPG signature of")),
1559 brown(file_name),
1560 )
1561 self._entropy.output(
1562 mytxt,
1563 level = "info",
1564 header = blue("\[email protected]@ "),
1565 back = True
1566 )
1567
1568 is_valid, err_msg = repo_sec.verify_file(self._repository_id,
1569 target_path, sign_path)
1570 if is_valid:
1571 mytxt = "%s: %s" % (
1572 darkgreen(_("Verified GPG signature of")),
1573 brown(file_name),
1574 )
1575 self._entropy.output(
1576 mytxt,
1577 level = "info",
1578 header = blue("\[email protected]@ ")
1579 )
1580 else:
1581 mytxt = "%s: %s" % (
1582 darkred(_("Error during GPG verification of")),
1583 file_name,
1584 )
1585 self._entropy.output(
1586 mytxt,
1587 level = "error",
1588 header = "\t%s " % (bold("!!!"),)
1589 )
1590 mytxt = "%s: %s" % (
1591 purple(_("It could mean a potential security risk")),
1592 err_msg,
1593 )
1594 self._entropy.output(
1595 mytxt,
1596 level = "error",
1597 header = "\t%s " % (bold("!!!"),)
1598 )
1599 gpg_rc = 1
1600
1601 return gpg_rc
1602
1604 """
1605 Update the local repository database through the webservice
1606 using a differential sync.
1607 """
1608 repo_db = None
1609 try:
1610 repo_db = self.__get_webserv_local_database()
1611 if repo_db is None:
1612 raise AttributeError()
1613 return self.__handle_webserv_database_sync(repo_db)
1614 except (DatabaseError, IntegrityError, OperationalError,
1615 AttributeError,):
1616 return False
1617 finally:
1618 if repo_db is not None:
1619 repo_db.commit()
1620 repo_db.close()
1621
1622 return False
1623
1625
1626 try:
1627 webserv = self._webservice
1628 except WebService.UnsupportedService as err:
1629 const_debug_write(__name__,
1630 "__handle_webserv_database_sync: error: %s" % (err,))
1631 return False
1632
1633 try:
1634 mypackage_ids = mydbconn.listAllPackageIds()
1635 except (DatabaseError, IntegrityError, OperationalError,):
1636 return False
1637
1638 added_ids, removed_ids = self.__get_webserv_database_differences(
1639 webserv, mypackage_ids)
1640 if (None in (added_ids, removed_ids)) or \
1641 (not added_ids and not removed_ids and self.__force):
1642
1643 return False
1644
1645 threshold = 100
1646
1647 if len(added_ids) > threshold:
1648 mytxt = "%s: %s (%s: %s/%s)" % (
1649 blue(_("Web Service")),
1650 darkred(_("skipping differential sync")),
1651 brown(_("threshold")),
1652 blue(str(len(added_ids))),
1653 darkred(str(threshold)),
1654 )
1655 self._entropy.output(
1656 mytxt,
1657 importance = 0,
1658 level = "info",
1659 header = blue(" # "),
1660 )
1661 return False
1662
1663 count = 0
1664 chunk_size = RepositoryWebService.MAXIMUM_PACKAGE_REQUEST_SIZE
1665 added_segments = []
1666 mytmp = []
1667
1668 for package_id in added_ids:
1669 count += 1
1670 mytmp.append(package_id)
1671 if count % chunk_size == 0:
1672 added_segments.append(mytmp[:])
1673 del mytmp[:]
1674 if mytmp:
1675 added_segments.append(mytmp[:])
1676
1677 def _do_fetch(fetch_sts_map, segment, count, maxcount):
1678 try:
1679 try:
1680 pkg_meta = webserv.get_packages_metadata(segment)
1681 except WebService.WebServiceException as err:
1682 const_debug_write(__name__,
1683 "__handle_webserv_database_sync: error: %s" % (err,))
1684 mytxt = "%s: %s" % (
1685 blue(_("Web Service communication error")),
1686 err,
1687 )
1688 self._entropy.output(
1689 mytxt, importance = 1, level = "info",
1690 header = "\t", count = (count, maxcount,)
1691 )
1692 fetch_sts_map['error'] = True
1693 return
1694 except KeyboardInterrupt:
1695 const_debug_write(__name__,
1696 "__handle_webserv_database_sync: keyboard interrupt")
1697 fetch_sts_map['error'] = True
1698 return
1699
1700 if not pkg_meta:
1701 const_debug_write(__name__,
1702 "__handle_webserv_database_sync: empty data: %s" % (
1703 pkg_meta,))
1704 self._entropy.output(
1705 _("Web Service data error"), importance = 1,
1706 level = "info", header = "\t",
1707 count = (count, maxcount,)
1708 )
1709 fetch_sts_map['error'] = True
1710 return
1711
1712 try:
1713 for package_id, pkg_data in pkg_meta.items():
1714 dumpobj(
1715 "%s%s" % (self.WEBSERV_CACHE_ID, package_id,),
1716 pkg_data,
1717 ignore_exceptions = False
1718 )
1719 except (IOError, EOFError, OSError,) as e:
1720 mytxt = "%s: %s: %s." % (
1721 blue(_("Local status")),
1722 darkred("Error storing data"),
1723 e,
1724 )
1725 self._entropy.output(
1726 mytxt, importance = 1, level = "info",
1727 header = "\t", count = (count, maxcount,)
1728 )
1729 fetch_sts_map['error'] = True
1730 return
1731
1732 return
1733 finally:
1734 fetch_sts_map['sem'].release()
1735
1736
1737
1738
1739 max_threads = 4
1740 fetch_sts_map = {
1741 'sem': threading.Semaphore(max_threads),
1742 'error': False,
1743 }
1744
1745
1746 count = 0
1747 maxcount = len(added_segments)
1748 product = self._settings['repositories']['product']
1749 segment = None
1750 threads = []
1751 for segment in added_segments:
1752 count += 1
1753 mytxt = "%s %s" % (blue(_("Fetching segments")), "...",)
1754 self._entropy.output(
1755 mytxt, importance = 0, level = "info",
1756 header = "\t", back = True, count = (count, maxcount,)
1757 )
1758 fetch_sts_map['sem'].acquire()
1759 if len(threads) >= max_threads:
1760 const_debug_write(__name__,
1761 purple("joining all the parallel threads"))
1762
1763
1764 for th in threads:
1765 th.join()
1766 const_debug_write(__name__, purple("parallel threads joined"))
1767 del threads[:]
1768 if fetch_sts_map['error']:
1769 return None
1770 th = ParallelTask(_do_fetch, fetch_sts_map, segment, count,
1771 maxcount)
1772 th.daemon = True
1773 th.start()
1774 threads.append(th)
1775
1776 for th in threads:
1777 th.join()
1778
1779 if fetch_sts_map['error']:
1780 return None
1781
1782 del added_segments
1783
1784
1785 repo_metadata = self.__get_webserv_repository_metadata()
1786
1787 if not repo_metadata:
1788 mytxt = "%s: %s" % (
1789 blue(_("Web Service status")),
1790 darkred(_("cannot fetch repository metadata")),
1791 )
1792 self._entropy.output(
1793 mytxt,
1794 importance = 0,
1795 level = "info",
1796 header = blue(" # "),
1797 )
1798 return None
1799
1800
1801 try:
1802 mydbconn.setRepositoryUpdatesDigest(self._repository_id,
1803 repo_metadata['treeupdates_digest'])
1804 mydbconn.bumpTreeUpdatesActions(
1805 repo_metadata['treeupdates_actions'])
1806 except (Error,):
1807 mytxt = "%s: %s" % (
1808 blue(_("Web Service status")),
1809 darkred(_("cannot update treeupdates data")),
1810 )
1811 self._entropy.output(
1812 mytxt,
1813 importance = 0,
1814 level = "info",
1815 header = blue(" # "),
1816 )
1817 return None
1818
1819
1820 try:
1821 mydbconn.clearPackageSets()
1822 mydbconn.insertPackageSets(repo_metadata['sets'])
1823 except (Error,):
1824 mytxt = "%s: %s" % (
1825 blue(_("Web Service status")),
1826 darkred(_("cannot update package sets data")),
1827 )
1828 self._entropy.output(
1829 mytxt,
1830 importance = 0,
1831 level = "info",
1832 header = blue(" # "),
1833 )
1834 return None
1835
1836
1837 for package_id in added_ids:
1838 mydata = self._cacher.pop("%s%s" % (self.WEBSERV_CACHE_ID,
1839 package_id,))
1840 if mydata is None:
1841 mytxt = "%s: %s" % (
1842 blue(_("Fetch error on segment while adding")),
1843 darkred(str(segment)),
1844 )
1845 self._entropy.output(
1846 mytxt, importance = 1, level = "warning",
1847 header = " "
1848 )
1849 return False
1850
1851 mytxt = "%s %s" % (
1852 darkgreen("++"),
1853 teal(mydata['atom']),
1854 )
1855 self._entropy.output(
1856 mytxt, importance = 0, level = "info",
1857 header = " ")
1858 try:
1859 mydbconn.addPackage(
1860 mydata, revision = mydata['revision'],
1861 package_id = package_id,
1862 formatted_content = True
1863 )
1864 except (Error,) as err:
1865 if const_debug_enabled():
1866 entropy.tools.print_traceback()
1867 self._entropy.output("%s: %s" % (
1868 blue(_("repository error while adding packages")),
1869 err,),
1870 importance = 1, level = "warning",
1871 header = " "
1872 )
1873 return False
1874
1875
1876
1877 atoms_map = dict((x, mydbconn.retrieveAtom(x),) for x in removed_ids)
1878 for package_id in removed_ids:
1879 myatom = atoms_map.get(package_id)
1880
1881 mytxt = "%s %s" % (
1882 darkred("--"),
1883 purple(str(myatom)),)
1884 self._entropy.output(
1885 mytxt, importance = 0, level = "info",
1886 header = " ")
1887 try:
1888 mydbconn.removePackage(package_id)
1889 except (Error,):
1890 self._entropy.output(
1891 blue(_("repository error while removing packages")),
1892 importance = 1, level = "warning",
1893 header = " "
1894 )
1895 return False
1896
1897 mydbconn.commit()
1898 mydbconn.clearCache()
1899
1900 result = False
1901 mychecksum = mydbconn.checksum(do_order = True,
1902 strict = False, include_signatures = True)
1903 if repo_metadata['checksum'] == mychecksum:
1904 result = True
1905 else:
1906 self._entropy.output(
1907 blue(_("Repository checksum doesn't match remote.")),
1908 importance = 0, level = "info", header = "\t",
1909 )
1910 mytxt = "%s: %s" % (_('local'), mychecksum,)
1911 self._entropy.output(
1912 mytxt, importance = 0,
1913 level = "info", header = "\t",
1914 )
1915 mytxt = "%s: %s" % (_('remote'), repo_metadata['checksum'],)
1916 self._entropy.output(
1917 mytxt, importance = 0,
1918 level = "info", header = "\t",
1919 )
1920
1921 return result
1922
1924 rev = self._remote_webservice_revision()
1925 if rev is not None:
1926 return rev
1927
1928 avail_data = self._settings['repositories']['available']
1929 repo_data = avail_data[self._repository_id]
1930
1931
1932 try:
1933 uri = repo_data['databases'][0]['uri']
1934 except IndexError:
1935 rev = -1
1936 return rev
1937
1938 basic_user = repo_data.get('username')
1939 basic_pwd = repo_data.get('password')
1940 https_validate_cert = not repo_data.get('https_validate_cert') == "false"
1941
1942 rev = self._remote_revision(url,
1943 http_basic_user = basic_user,
1944 http_basic_pwd = basic_pwd,
1945 https_validate_cert = https_validate_cert)
1946
1947 return rev
1948
1950 """
1951 Return the remote repository revision using the webservice.
1952 This method returns None if the request failed or the
1953 webservice is not available.
1954 """
1955 if self._repo_eapi < 3:
1956 return
1957
1958
1959 revision = self.__get_webserv_repository_revision()
1960 if revision is not None:
1961 try:
1962 revision = int(revision)
1963 except ValueError:
1964 revision = None
1965
1966 if revision is not None:
1967 return revision
1968
1969
1970 self._repo_eapi -= 1
1971
1972 - def _remote_revision(self, uri, http_basic_user = None,
1973 http_basic_pwd = None,
1974 https_validate_cert = True):
1975 """
1976 Return the remote repository revision by downloading
1977 the revision file from the given uri.
1978 """
1979 sep = const_convert_to_unicode("/")
1980 url = uri + sep + etpConst['etpdatabaserevisionfile']
1981
1982 tmp_fd, tmp_path = None, None
1983 rev = "-1"
1984 try:
1985 tmp_fd, tmp_path = const_mkstemp(
1986 prefix = "AvailableEntropyRepository.remote_revision")
1987 fetcher = self._entropy._url_fetcher(
1988 url, tmp_path, resume = False,
1989 http_basic_user = http_basic_user,
1990 http_basic_pwd = http_basic_pwd,
1991 https_validate_cert = https_validate_cert)
1992 fetch_rc = fetcher.download()
1993 if fetch_rc not in self.FETCH_ERRORS:
1994 with codecs.open(tmp_path, "r") as tmp_f:
1995 rev = tmp_f.readline().strip()
1996 except (IOError, OSError):
1997
1998 pass
1999 finally:
2000 if tmp_fd is not None:
2001 try:
2002 os.close(tmp_fd)
2003 except OSError:
2004 pass
2005 if tmp_path is not None:
2006 try:
2007 os.remove(tmp_path)
2008 except OSError:
2009 pass
2010
2011
2012 try:
2013 rev = int(rev)
2014 except ValueError:
2015
2016 rev = -1
2017
2018 return rev
2019
2021
2022
2023 if not entropy.tools.is_root():
2024 raise PermissionDenied(
2025 "cannot update repository as unprivileged user")
2026
2027 self._show_repository_information()
2028
2029 selected = self._select_database_mirror()
2030 if selected is None:
2031 return EntropyRepositoryBase.REPOSITORY_NOT_AVAILABLE
2032 revision, uri, cformat = selected
2033
2034 updatable = self._is_repository_updatable(revision)
2035 if not self.__force:
2036 if not updatable:
2037 mytxt = "%s: %s." % (bold(_("Attention")),
2038 red(_("repository is already up to date")),)
2039 self._entropy.output(
2040 mytxt,
2041 importance = 1,
2042 level = "info",
2043 header = "\t"
2044 )
2045 return EntropyRepositoryBase.REPOSITORY_ALREADY_UPTODATE
2046
2047 locked = not self._is_repository_unlocked(uri)
2048 if locked:
2049 mytxt = "%s: %s. %s." % (
2050 bold(_("Attention")),
2051 red(_("Repository is being updated")),
2052 red(_("Try again in a few minutes")),
2053 )
2054 self._entropy.output(
2055 mytxt,
2056 importance = 1,
2057 level = "warning",
2058 header = "\t"
2059 )
2060 return EntropyRepositoryBase.REPOSITORY_NOT_AVAILABLE
2061
2062
2063 self._ensure_repository_path()
2064
2065
2066
2067 db_checksum_down_status = False
2068 do_db_update_transfer = False
2069 rc = 0
2070
2071 my_repos = self._settings['repositories']
2072 avail_data = my_repos['available']
2073 repo_data = avail_data[self._repository_id]
2074
2075
2076 dumpfile = os.path.join(repo_data['dbpath'],
2077 etpConst['etpdatabasedumplight'])
2078 dbfile = os.path.join(repo_data['dbpath'],
2079 etpConst['etpdatabasefile'])
2080 dbfile_old = dbfile+".sync"
2081 cmethod = etpConst['etpdatabasecompressclasses'].get(
2082 cformat)
2083
2084 while True:
2085
2086 downloaded_db_item = None
2087 sig_down_status = False
2088 db_checksum_down_status = False
2089 if self._repo_eapi < 3:
2090
2091 down_status, sig_down_status, downloaded_db_item = \
2092 self.__database_download(uri, cmethod)
2093 if not down_status:
2094 return EntropyRepositoryBase.REPOSITORY_NOT_AVAILABLE
2095 db_checksum_down_status = \
2096 self.__database_checksum_download(uri, cmethod)
2097 break
2098
2099 elif self._repo_eapi == 3 and not const_file_writable(dbfile):
2100 do_db_update_transfer = None
2101 self._repo_eapi -= 1
2102 continue
2103
2104 elif self._repo_eapi == 3:
2105
2106 status = False
2107 try:
2108 status = self._webservice_database_sync()
2109 except:
2110
2111 entropy.tools.print_traceback()
2112 self.__remove_repository_files()
2113 raise
2114
2115 if not status:
2116
2117 do_db_update_transfer = None
2118 self._repo_eapi -= 1
2119 continue
2120
2121 break
2122
2123 downloaded_files = self._standard_items_download(uri)
2124
2125
2126 if downloaded_db_item is not None:
2127
2128 durl, dpath = self._construct_paths(
2129 uri, downloaded_db_item, cmethod)
2130 downloaded_files.append(dpath)
2131 if sig_down_status:
2132 d_sig_path = self.__append_gpg_signature_to_path(dpath)
2133 downloaded_files.append(d_sig_path)
2134
2135
2136
2137
2138 if not db_checksum_down_status:
2139
2140 self.__remove_repository_files()
2141 return EntropyRepositoryBase.REPOSITORY_NOT_AVAILABLE
2142
2143 rc = self._check_downloaded_database(uri, cmethod)
2144 if rc != 0:
2145
2146 self.__remove_repository_files()
2147 return EntropyRepositoryBase.REPOSITORY_CHECKSUM_ERROR
2148
2149
2150 if self._gpg_feature:
2151 gpg_available = self._install_gpg_key_if_available()
2152 if gpg_available:
2153 gpg_rc = self._gpg_verify_downloaded_files(downloaded_files)
2154
2155
2156 files_to_remove = []
2157 if self._repo_eapi in (1, 2,):
2158
2159
2160 if (do_db_update_transfer is not None) and not \
2161 do_db_update_transfer:
2162
2163 try:
2164 os.rename(dbfile, dbfile_old)
2165 do_db_update_transfer = True
2166 except OSError as err:
2167 const_debug_write(
2168 __name__, "rename failed: %s" % (err,))
2169 do_db_update_transfer = False
2170
2171 unpack_status, unpacked_item = \
2172 self._downloaded_database_unpack(uri, cmethod)
2173
2174 if not unpack_status:
2175
2176 self.__remove_repository_files()
2177 return EntropyRepositoryBase.REPOSITORY_GENERIC_ERROR
2178
2179 unpack_url, unpack_path = self._construct_paths(
2180 uri, unpacked_item, cmethod)
2181 files_to_remove.append(unpack_path)
2182
2183
2184 if not os.path.isfile(dbfile):
2185 do_db_update_transfer = False
2186
2187 elif os.path.isfile(dbfile) and not do_db_update_transfer and \
2188 (self._repo_eapi != 1):
2189 os.remove(dbfile)
2190
2191 if self._repo_eapi == 2:
2192 rc = self.__eapi2_inject_downloaded_dump(dumpfile,
2193 dbfile, cmethod)
2194
2195 if do_db_update_transfer:
2196 self.__eapi1_eapi2_databases_alignment(dbfile, dbfile_old)
2197
2198 if self._repo_eapi == 2:
2199
2200 files_to_remove.append(dumpfile)
2201
2202 if rc != 0:
2203
2204 self.__remove_repository_files()
2205 files_to_remove.append(dbfile_old)
2206 for path in files_to_remove:
2207 try:
2208 os.remove(path)
2209 except OSError:
2210 continue
2211 return EntropyRepositoryBase.REPOSITORY_GENERIC_ERROR
2212
2213
2214
2215 downloaded_files.append(dbfile)
2216 for downloaded_file in sorted(set(downloaded_files)):
2217 try:
2218 const_setup_file(downloaded_file,
2219 etpConst['entropygid'], 0o644,
2220 uid = etpConst['uid'])
2221 except (OSError, IOError) as err:
2222 if err.errno != errno.ENOENT:
2223 raise
2224
2225
2226 for path in files_to_remove:
2227 try:
2228 os.remove(path)
2229 except OSError:
2230 continue
2231
2232 valid = self.__validate_database()
2233 if not valid:
2234
2235 return EntropyRepositoryBase.REPOSITORY_GENERIC_ERROR
2236
2237 self.__update_repository_revision(revision)
2238 if self._entropy._indexing:
2239 self.__database_indexing()
2240
2241 try:
2242 spm_class = self._entropy.Spm_class()
2243 spm_class.entropy_client_post_repository_update_hook(
2244 self._entropy, self._repository_id)
2245 except Exception as err:
2246 entropy.tools.print_traceback()
2247 mytxt = "%s: %s" % (
2248 blue(_("Configuration files update error, "
2249 "not critical, continuing")),
2250 err,
2251 )
2252 self._entropy.output(mytxt, importance = 0,
2253 level = "info", header = blue(" # "),)
2254
2255
2256 try:
2257 os.remove(dbfile_old)
2258 except OSError:
2259 pass
2260
2261 return EntropyRepositoryBase.REPOSITORY_UPDATED_OK
2262
2265 """
2266 Objects inheriting from this class support package masking.
2267 A masked package is a package that is not visible to user and thus not
2268 selectable in dependency calculation and also not directly installable.
2269 The only repositories that need to support the feature are those containing
2270 installable packages, like AvailablePackagesRepository.
2271 """
2272
2273 _real_client_settings = None
2274 _real_client_settings_lock = threading.Lock()
2275
2278
2279 @property
2281 """
2282 Load the Entropy Client settings object.
2283 """
2284 if self._real_client_settings is None:
2285 with self._real_client_settings_lock:
2286 if self._real_client_settings is None:
2287 from entropy.client.interfaces import Client
2288
2289 self._real_client_settings = Client().ClientSettings()
2290
2291 return self._real_client_settings
2292
2293 @property
2295 """
2296 Get the ClientSystemSettingsPlugin instance from Entropy Client.
2297 """
2298 from entropy.client.interfaces import Client
2299 return Client()._settings_client_plugin
2300
2302 if self._caching:
2303 return loadobj(
2304 "MaskableRepositoryFilter/%s_%s/%s" % (
2305 self.name,
2306 self.atomMatchCacheKey(),
2307 package_id,
2308 )
2309 )
2310
2312 if self._caching:
2313 dumpobj(
2314 "MaskableRepositoryFilter/%s_%s/%s" % (
2315 self.name,
2316 self.atomMatchCacheKey(),
2317 package_id,
2318 ),
2319 value)
2320
2322
2323 ref = self._settings['pkg_masking_reference']
2324 if (package_id, self.name) in \
2325 self._settings['live_packagemasking']['mask_matches']:
2326
2327
2328 return -1, ref['user_live_mask']
2329
2330 elif (package_id, self.name) in \
2331 self._settings['live_packagemasking']['unmask_matches']:
2332
2333 return package_id, ref['user_live_unmask']
2334
2336
2337 with self._settings['mask']:
2338
2339 cache_obj = self._settings['mask'].get()
2340 if cache_obj is None:
2341 cache_obj = {}
2342 self._settings['mask'].set(cache_obj)
2343 user_package_mask_ids = cache_obj.get(self.name)
2344
2345 if user_package_mask_ids is None:
2346 user_package_mask_ids = set()
2347
2348 for atom in self._settings['mask']:
2349 atom, repository_ids = entropy.dep.dep_get_match_in_repos(atom)
2350 if repository_ids is not None:
2351 if self.name not in repository_ids:
2352
2353 continue
2354
2355 matches, r = self.atomMatch(atom, multiMatch = True,
2356 maskFilter = False)
2357 if r != 0:
2358 continue
2359 user_package_mask_ids |= set(matches)
2360
2361 cache_obj[self.name] = user_package_mask_ids
2362
2363 if package_id in user_package_mask_ids:
2364
2365 ref = self._settings['pkg_masking_reference']
2366 myr = ref['user_package_mask']
2367
2368 try:
2369 clset = self._client_settings
2370 validator_cache = clset['masking_validation']['cache']
2371 validator_cache[(package_id, self.name, live)] = -1, myr
2372 except KeyError:
2373 pass
2374
2375 return -1, myr
2376
2378
2379 with self._settings['unmask']:
2380
2381 cache_obj = self._settings['unmask'].get()
2382 if cache_obj is None:
2383 cache_obj = {}
2384 self._settings['unmask'].set(cache_obj)
2385 user_package_unmask_ids = cache_obj.get(self.name)
2386
2387 if user_package_unmask_ids is None:
2388
2389 user_package_unmask_ids = set()
2390 for atom in self._settings['unmask']:
2391 atom, repository_ids = entropy.dep.dep_get_match_in_repos(atom)
2392 if repository_ids is not None:
2393 if self.name not in repository_ids:
2394
2395 continue
2396 matches, r = self.atomMatch(atom, multiMatch = True,
2397 maskFilter = False)
2398 if r != 0:
2399 continue
2400 user_package_unmask_ids |= set(matches)
2401
2402 cache_obj[self.name] = user_package_unmask_ids
2403
2404 if package_id in user_package_unmask_ids:
2405
2406 ref = self._settings['pkg_masking_reference']
2407 myr = ref['user_package_unmask']
2408 try:
2409 clset = self._client_settings
2410 validator_cache = clset['masking_validation']['cache']
2411 validator_cache[(package_id, self.name, live)] = \
2412 package_id, myr
2413 except KeyError:
2414 pass
2415
2416 return package_id, myr
2417
2419
2420
2421 repos_mask = {}
2422 clset = self._client_settings
2423 if clset:
2424 repos_mask = clset['repositories']['mask']
2425
2426 repomask = repos_mask.get(self.name)
2427 if isinstance(repomask, (list, set, frozenset)):
2428
2429
2430
2431 mask_repo_id = "%[email protected]@:of:%s" % (self.name, self.name,)
2432 repomask_ids = repos_mask.get(mask_repo_id)
2433
2434 if not isinstance(repomask_ids, set):
2435 repomask_ids = set()
2436 for atom in repomask:
2437 matches, r = self.atomMatch(atom, multiMatch = True,
2438 maskFilter = False)
2439 if r != 0:
2440 continue
2441 repomask_ids |= set(matches)
2442 repos_mask[mask_repo_id] = repomask_ids
2443
2444 if package_id in repomask_ids:
2445
2446 ref = self._settings['pkg_masking_reference']
2447 myr = ref['repository_packages_db_mask']
2448
2449 try:
2450 validator_cache = clset['masking_validation']['cache']
2451 validator_cache[(package_id, self.name, live)] = \
2452 -1, myr
2453 except KeyError:
2454 pass
2455
2456 return -1, myr
2457
2459
2460 if not self._settings['license_mask']:
2461 return
2462
2463 mylicenses = self.retrieveLicense(package_id)
2464 mylicenses = mylicenses.strip().split()
2465 lic_mask = self._settings['license_mask']
2466 for mylicense in mylicenses:
2467
2468 if mylicense not in lic_mask:
2469 continue
2470
2471 ref = self._settings['pkg_masking_reference']
2472 myr = ref['user_license_mask']
2473 try:
2474 clset = self._client_settings
2475 validator_cache = clset['masking_validation']['cache']
2476 validator_cache[(package_id, self.name, live)] = -1, myr
2477 except KeyError:
2478 pass
2479
2480 return -1, myr
2481
2483
2484
2485
2486
2487 mykeywords = self.retrieveKeywords(package_id)
2488 if mykeywords == set([""]):
2489 mykeywords = set(['**'])
2490
2491 mask_ref = self._settings['pkg_masking_reference']
2492
2493
2494
2495 same_keywords = etpConst['keywords'] & mykeywords
2496 if same_keywords:
2497 myr = mask_ref['system_keyword']
2498 try:
2499 clset = self._client_settings
2500 validator_cache = clset['masking_validation']['cache']
2501 validator_cache[(package_id, self.name, live)] = \
2502 package_id, myr
2503
2504 except KeyError:
2505 pass
2506
2507 return package_id, myr
2508
2509
2510
2511
2512
2513 keyword_repo = self._settings['keywords']['repositories']
2514
2515 for keyword in tuple(keyword_repo.get(self.name, {}).keys()):
2516
2517 if keyword not in mykeywords:
2518 continue
2519
2520 keyword_data = keyword_repo[self.name].get(keyword)
2521 if not keyword_data:
2522 continue
2523
2524 if "*" in keyword_data:
2525
2526 myr = mask_ref['user_repo_package_keywords_all']
2527 try:
2528 clset = self._client_settings
2529 validator_cache = clset['masking_validation']['cache']
2530 validator_cache[(package_id, self.name, live)] = \
2531 package_id, myr
2532 except KeyError:
2533 pass
2534
2535 return package_id, myr
2536
2537 kwd_key = "%s_ids" % (keyword,)
2538 keyword_data_ids = keyword_repo[self.name].get(kwd_key)
2539 if not isinstance(keyword_data_ids, set):
2540
2541 keyword_data_ids = set()
2542 for atom in keyword_data:
2543 matches, r = self.atomMatch(atom, multiMatch = True,
2544 maskFilter = False)
2545 if r != 0:
2546 continue
2547 keyword_data_ids |= matches
2548
2549 keyword_repo[self.name][kwd_key] = keyword_data_ids
2550
2551 if package_id in keyword_data_ids:
2552
2553 myr = mask_ref['user_repo_package_keywords']
2554 try:
2555 clset = self._client_settings
2556 validator_cache = clset['masking_validation']['cache']
2557 validator_cache[(package_id, self.name, live)] = \
2558 package_id, myr
2559 except KeyError:
2560 pass
2561 return package_id, myr
2562
2563 keyword_pkg = self._settings['keywords']['packages']
2564
2565
2566
2567 for keyword in keyword_pkg.keys():
2568
2569
2570
2571 if keyword not in mykeywords:
2572 continue
2573
2574 keyword_data = keyword_pkg.get(keyword)
2575 if not keyword_data:
2576 continue
2577
2578 kwd_key = "%s_ids" % (keyword,)
2579 keyword_data_ids = keyword_pkg.get(self.name+kwd_key)
2580
2581 if not isinstance(keyword_data_ids, (list, set)):
2582 keyword_data_ids = set()
2583 for atom in keyword_data:
2584
2585 matches, r = self.atomMatch(atom, multiMatch = True,
2586 maskFilter = False)
2587 if r != 0:
2588 continue
2589 keyword_data_ids |= matches
2590
2591 keyword_pkg[self.name+kwd_key] = keyword_data_ids
2592
2593 if package_id in keyword_data_ids:
2594
2595
2596 myr = mask_ref['user_package_keywords']
2597 try:
2598 clset = self._client_settings
2599 validator_cache = clset['masking_validation']['cache']
2600 validator_cache[(package_id, self.name, live)] = \
2601 package_id, myr
2602 except KeyError:
2603 pass
2604
2605 return package_id, myr
2606
2607
2608
2609
2610
2611
2612
2613 clset = self._client_settings
2614 if clset is None:
2615
2616 return
2617
2618
2619 repo_keywords = clset['repositories']['repos_keywords'].get(
2620 self.name)
2621 if repo_keywords is None:
2622
2623 return
2624
2625
2626 same_keywords = repo_keywords.get('universal') & mykeywords
2627 if same_keywords:
2628
2629 myr = mask_ref['repository_packages_db_keywords']
2630 validator_cache = clset['masking_validation']['cache']
2631 validator_cache[(package_id, self.name, live)] = \
2632 package_id, myr
2633 return package_id, myr
2634
2635
2636
2637 repo_settings = repo_keywords.get('packages')
2638 if not repo_settings:
2639
2640 return
2641
2642 cached_key = "packages_ids"
2643 keyword_data_ids = repo_keywords.get(cached_key)
2644 if not isinstance(keyword_data_ids, dict):
2645
2646
2647 keyword_data_ids = {}
2648 for atom, values in repo_settings.items():
2649 matches, r = self.atomMatch(atom, multiMatch = True,
2650 maskFilter = False)
2651 if r != 0:
2652 continue
2653 for match in matches:
2654 obj = keyword_data_ids.setdefault(match, set())
2655 obj.update(values)
2656
2657 repo_keywords[cached_key] = keyword_data_ids
2658
2659 pkg_keywords = keyword_data_ids.get(package_id, set())
2660 if "**" in pkg_keywords:
2661 same_keywords = True
2662 else:
2663 same_keywords = pkg_keywords & etpConst['keywords']
2664 if same_keywords:
2665
2666 myr = mask_ref['repository_packages_db_keywords']
2667 validator_cache = clset['masking_validation']['cache']
2668 validator_cache[(package_id, self.name, live)] = \
2669 package_id, myr
2670 return package_id, myr
2671
2673 """
2674 Reimplemented from EntropyRepositoryBase
2675 """
2676 validator_cache = self._client_settings.get(
2677 'masking_validation', {}).get('cache', {})
2678
2679 cached = validator_cache.get((package_id, self.name, live))
2680 if cached is not None:
2681 return cached
2682
2683
2684 cached = self._mask_filter_fetch_cache(package_id)
2685 if cached is not None:
2686 return cached
2687
2688
2689 if len(validator_cache) > 100000:
2690 validator_cache.clear()
2691
2692 if live:
2693 data = self._maskFilter_live(package_id)
2694 if data:
2695 return data
2696
2697 data = self._maskFilter_user_package_mask(package_id, live)
2698 if data:
2699 self._mask_filter_store_cache(package_id, data)
2700 return data
2701
2702 data = self._maskFilter_user_package_unmask(package_id, live)
2703 if data:
2704 self._mask_filter_store_cache(package_id, data)
2705 return data
2706
2707 data = self._maskFilter_packages_db_mask(package_id, live)
2708 if data:
2709 self._mask_filter_store_cache(package_id, data)
2710 return data
2711
2712 data = self._maskFilter_package_license_mask(package_id, live)
2713 if data:
2714 self._mask_filter_store_cache(package_id, data)
2715 return data
2716
2717 data = self._maskFilter_keyword_mask(package_id, live)
2718 if data:
2719 self._mask_filter_store_cache(package_id, data)
2720 return data
2721
2722
2723 myr = self._settings['pkg_masking_reference']['completely_masked']
2724 validator_cache[(package_id, self.name, live)] = -1, myr
2725 self._mask_filter_store_cache(package_id, data)
2726 return -1, myr
2727
2736
2739 """
2740 This class represents the available packages repository and is a direct
2741 subclass of EntropyRepository. It implements the update() method in order
2742 to make possible to update the repository.
2743 """
2751
2752 @staticmethod
2753 - def update(entropy_client, repository_id, force, gpg):
2765
2766 @staticmethod
2768 """
2769 Reimplemented from EntropyRepositoryBase
2770 """
2771 db_data = SystemSettings()['repositories']['available'][repository_id]
2772 fname = os.path.join(db_data['dbpath'],
2773 etpConst['etpdatabaserevisionfile'])
2774
2775 revision = -1
2776 enc = etpConst['conf_encoding']
2777 try:
2778 with codecs.open(fname, "r", encoding=enc) as f:
2779 read_data = f.readline().strip()
2780 try:
2781 revision = int(read_data)
2782 except ValueError:
2783 pass
2784 except (OSError, IOError) as err:
2785 if err.errno != errno.ENOENT:
2786 raise
2787
2788 return revision
2789
2790 @staticmethod
2797
2798 - def handlePackage(self, pkg_data, revision = None,
2799 formattedContent = False):
2800 """
2801 Reimplemented from EntropyRepository
2802 """
2803 raise PermissionDenied(
2804 "cannot execute handlePackage on this repository")
2805
2806 - def addPackage(self, pkg_data, revision = -1, package_id = None,
2807 formatted_content = False):
2808 """
2809 Reimplemented from EntropyRepository
2810 """
2811 raise PermissionDenied(
2812 "cannot execute addPackage on this repository")
2813
2815 """
2816 Reimplemented from EntropyRepository
2817 """
2818 raise PermissionDenied(
2819 "cannot execute removePackage on this repository")
2820
2822
2823 try:
2824 clset = self._client_settings
2825 except AttributeError:
2826
2827
2828
2829 pass
2830 else:
2831 clset.get('masking_validation', {}).get('cache', {}).clear()
2832 EntropyRepository.clearCache(self)
2833
2836 """
2837 This class represents a generic packages repository and is a direct
2838 subclass of EntropyRepository.
2839 Even GenericRepository is a CachedRepository because its object could
2840 get cached by 3rd party. Actually, we require this because our installed
2841 packages repository could end up being a GenericRepository, when running
2842 in fail-safe mode.
2843 """
2844
2845 - def handlePackage(self, pkg_data, revision = None,
2846 formattedContent = False):
2847 """
2848 Reimplemented from EntropyRepository.
2849 It is supposed that a generic repository should not support
2850 handlePackage. You can override this (at your own risk) by setting the
2851 "override_handlePackage" property to True. In this case, a generic
2852 addPackage() call is issued.
2853 """
2854 override = getattr(self, 'override_handlePackage', False)
2855 if not override:
2856 raise PermissionDenied(
2857 "cannot execute handlePackage on this repository")
2858
2859 return self.addPackage(pkg_data, revision = revision,
2860 formatted_content = formattedContent)
2861
2863 """
2864 Reimplemented from EntropyRepository.
2865 It is supposed that a generic repository doesn't support package
2866 masking. You can override this by setting the "enable_mask_filter"
2867 to True.
2868 """
2869 enabled = getattr(self, 'enable_mask_filter', False)
2870 if not enabled:
2871 return package_id, 0
2872 return MaskableRepository.maskFilter(self, package_id, live = live)
2873