Package entropy :: Package services :: Module client

Source Code for Module entropy.services.client

   1  # -*- coding: utf-8 -*- 
   2  """ 
   3   
   4      @author: Fabio Erculiani <[email protected]> 
   5      @contact: [email protected] 
   6      @copyright: Fabio Erculiani 
   7      @license: GPL-2 
   8   
   9      B{Entropy Base Repository Web Services client interface}. 
  10   
  11  """ 
  12  __all__ = ["WebServiceFactory", "WebService"] 
  13   
  14  import sys 
  15  import os 
  16  import errno 
  17  import json 
  18  import threading 
  19  import hashlib 
  20  import ssl 
  21  import socket 
  22   
  23  from entropy.const import const_is_python3, const_convert_to_rawstring, \ 
  24      const_get_int, const_mkstemp, const_dir_writable 
  25   
  26  if const_is_python3(): 
  27      import http.client as httplib 
  28      from io import StringIO 
  29      import urllib.parse as urllib_parse 
  30  else: 
  31      import httplib 
  32      from cStringIO import StringIO 
  33      import urllib as urllib_parse 
  34   
  35  import entropy.dump 
  36  from entropy.core import Singleton 
  37  from entropy.cache import EntropyCacher 
  38  from entropy.const import const_debug_write, const_setup_file, etpConst, \ 
  39      const_convert_to_rawstring, const_isunicode, const_isstring, \ 
  40      const_convert_to_unicode, const_isstring, const_debug_enabled 
  41  from entropy.core.settings.base import SystemSettings 
  42  from entropy.exceptions import EntropyException 
  43  import entropy.tools 
  44  import entropy.dep 
45 46 47 -class WebServiceFactory(object):
48 """ 49 Base Entropy Repository Web Services Factory. Generates 50 WebService objects that can be used to communicate with the established 51 web service. 52 This is a base class and subclasses should be preferred (example: 53 entropy.client.services.interfaces.ClientWebServiceFactory) 54 """
55 - class InvalidWebServiceFactory(EntropyException):
56 """ 57 Raised when an invalid WebService based class is passed. 58 """
59
60 - def __init__(self, web_service_class, entropy_client):
61 """ 62 WebServiceFactory constructor. 63 64 @param entropy_client: Entropy Client interface 65 @type entropy_client: entropy.client.interfaces.client.Client 66 """ 67 object.__init__(self) 68 if not issubclass(web_service_class, WebService): 69 raise WebServiceFactory.InvalidWebServiceFactory( 70 "invalid web_service_class") 71 self._entropy = entropy_client 72 self._service_class = web_service_class
73
74 - def new(self, repository_id):
75 """ 76 Return a new WebService object for given repository identifier. 77 78 @param repository_id: repository identifier 79 @rtype repository_id: string 80 @raise WebService.UnsupportedService: if web service is 81 explicitly unsupported by repository 82 """ 83 return self._service_class(self._entropy, repository_id)
84
85 86 -class WebService(object):
87 """ 88 This is the Entropy Repository Web Services that proxies requests over 89 an Web Services answering to HTTP POST requests 90 (either over HTTP or over HTTPS) in the following form: 91 92 Given that repositories must ship with a file (in their repository 93 meta file "packages.db.meta", coming from the server-side repository 94 directory) called "packages.db.webservices" 95 (etpConst['etpdatabasewebservicesfile']) containing the HTTP POST base 96 URL (example: http://packages.sabayon.org/api). 97 The POST url is composed as follows: 98 <base_url>/<method name> 99 Function arguments are then appended in JSON format, so that float, int, 100 strings and lists are correctly represented. 101 So, for example, if WebService exposes a method with the following 102 signature (with the base URL in example above): 103 104 float get_vote(string package_name) 105 106 The URL referenced will be: 107 108 http://packages.sabayon.org/api/get_vote 109 110 And the JSON dictionary will contain a key called "package_name" with 111 package_name value. 112 For methods requiring authentication, the JSON object will contain 113 "username" and "password" fields (clear text, so make sure to use HTTPS). 114 115 The Response depends on each specific method and it is given in JSON format 116 too, that is afterwards interpreted by the caller function, that will 117 always return the expected data format (see respective API documentation). 118 For more information about how to implement the Web Service, please see 119 the packages.git Sabayon repository, which contains a Pylons MVC web app. 120 In general, every JSON response must provide a 'code' field, representing 121 an HTTP-response alike return code (200 is ok, 500 is server error, 400 is 122 bad request, etc) and a 'message' field, containing the error message (if 123 no error, 'message' is usually empty). The RPC result is put inside the 124 'r' field. 125 126 This is a base class, and you should really implement a subclass providing 127 your own API methods, and use _method_getter(). 128 """ 129 130 # Supported communcation protocols 131 SUPPORTED_URL_SCHEMAS = ("http", "https") 132 133 # package icon metadata identifier 134 PKG_ICON_IDENTIFIER = "__icon__" 135 136 # Currently supported Web Service API level 137 # an API level defines a set of available remote calls and their data 138 # structure 139 SUPPORTED_API_LEVEL = 1 140 141 # Default common Web Service responses, please use these when 142 # implementing your web service 143 WEB_SERVICE_RESPONSE_CODE_OK = 200 144 WEB_SERVICE_INVALID_CREDENTIALS_CODE = 450 145 WEB_SERVICE_INVALID_REQUEST_CODE = 400 146 WEB_SERVICE_NOT_FOUND_CODE = 404 147 WEB_SERVICE_RESPONSE_ERROR_CODE = 503 148 149
150 - class WebServiceException(EntropyException):
151 """ 152 Base WebService exception class. 153 """
154 - def __init__(self, value, method = None, message = None):
155 self.value = value 156 self.method = method 157 self.message = message 158 Exception.__init__(self)
159
160 - def __get_method(self):
161 if self.method is None: 162 method = const_convert_to_unicode("") 163 else: 164 method = const_convert_to_unicode(self.method) 165 return method
166
167 - def __get_message(self):
168 if self.message is None: 169 message = const_convert_to_unicode("") 170 else: 171 message = const_convert_to_unicode(self.message) 172 return message
173
174 - def __unicode__(self):
175 method = self.__get_method() 176 message = self.__get_message() 177 if const_isstring(self.value): 178 return const_convert_to_unicode(method + " " + self.value) \ 179 + ", " + message 180 return const_convert_to_unicode(method + " " + repr(self.value)) \ 181 + ", " + message
182
183 - def __str__(self):
184 method = self.__get_method() 185 message = self.__get_message() 186 if const_isstring(self.value): 187 return method + " " + self.value + ", " + message 188 return method + " " + repr(self.value) + ", " + message
189
190 - class UnsupportedService(WebServiceException):
191 """ 192 Raised when Repository doesn't seem to support any Web Service 193 feature. 194 """
195
196 - class UnsupportedParameters(WebServiceException):
197 """ 198 Raised when input parameters cannot be converted to JSON. 199 Probably due to invalid input data. 200 """
201
202 - class RequestError(WebServiceException):
203 """ 204 If the request cannot be satisfied by the remote web service. 205 """
206
207 - class AuthenticationRequired(WebServiceException):
208 """ 209 When a method requiring valid user credentials is called without 210 being logged in. 211 """
212
213 - class AuthenticationFailed(WebServiceException):
214 """ 215 When credentials are stored locally but don't seem to work against 216 the Web Service. 217 """
218
219 - class MethodNotAvailable(WebServiceException):
220 """ 221 When calling a remote method that is not available. 222 """
223
224 - class MalformedResponse(WebServiceException):
225 """ 226 If JSON response cannot be converted back to dict. 227 """
228
229 - class UnsupportedAPILevel(WebServiceException):
230 """ 231 If this client and the Web Service expose a different API level. 232 """
233
234 - class MethodResponseError(WebServiceException):
235 """ 236 If the request has been accepted, but its computation stopped for 237 some reason. The encapsulated data contains the error code. 238 """
239
240 - class CacheMiss(WebServiceException):
241 """ 242 If the request is not available in the on-disk cache. 243 """
244
245 - def __init__(self, entropy_client, repository_id):
246 """ 247 WebService constructor. 248 NOTE: This base class must NOT use any Entropy Client specific method 249 and MUST rely on what is provided by it's parent class TextInterface. 250 251 @param entropy_client: Entropy Client interface 252 @type entropy_client: entropy.client.interfaces.client.Client 253 @param repository_id: repository identifier 254 @rtype repository_id: string 255 """ 256 self._cache_dir_lock = threading.RLock() 257 self._transfer_callback = None 258 self._entropy = entropy_client 259 self._repository_id = repository_id 260 self.__auth_storage = None 261 self.__settings = None 262 self.__cacher = None 263 self._default_timeout_secs = 10.0 264 self.__credentials_validated = False 265 # if this is set, cache will be considered invalid if older than 266 self._cache_aging_days = None 267 268 config = self.config(repository_id) 269 if config is None: 270 raise WebService.UnsupportedService("unsupported service [1]") 271 272 remote_url = config['url'] 273 if remote_url is None: 274 raise WebService.UnsupportedService("unsupported service [2]") 275 url_obj = config['_url_obj'] 276 277 self._request_url = remote_url 278 self._request_protocol = url_obj.scheme 279 self._request_host = url_obj.netloc 280 self._request_path = url_obj.path 281 self._config = config 282 283 const_debug_write(__name__, "WebService loaded, url: %s" % ( 284 self._request_url,))
285 286 @classmethod
287 - def config(cls, repository_id):
288 """ 289 Return the WebService configuration for the given repository. 290 The object returned is a dictionary containing the following 291 items: 292 - url: the Entropy WebService base URL (or None, if not supported) 293 - update_eapi: the maximum supported EAPI for repository updates. 294 - repo_eapi: the maximum supported EAPI for User Generate Content. 295 296 @param repository_id: repository identifier 297 @type repository_id: string 298 """ 299 settings = SystemSettings() 300 _repository_data = settings['repositories']['available'].get( 301 repository_id) 302 if _repository_data is None: 303 const_debug_write(__name__, "WebService.config error: no repo") 304 return None 305 306 web_services_conf = _repository_data.get('webservices_config') 307 if web_services_conf is None: 308 const_debug_write(__name__, "WebService.config error: no metadata") 309 return None 310 311 data = { 312 'url': None, 313 '_url_obj': None, 314 'update_eapi': None, 315 'repo_eapi': None, 316 } 317 318 content = [] 319 try: 320 content += entropy.tools.generic_file_content_parser( 321 web_services_conf, encoding = etpConst['conf_encoding']) 322 except (OSError, IOError) as err: 323 const_debug_write(__name__, "WebService.config error: %s" % ( 324 err,)) 325 return None 326 327 if not content: 328 const_debug_write( 329 __name__, "WebService.config error: empty config") 330 return None 331 332 remote_url = content.pop(0) 333 if remote_url == "-": # as per specs 334 remote_url = None 335 elif not remote_url: 336 remote_url = None 337 data['url'] = remote_url 338 339 if data['url']: 340 url_obj = entropy.tools.spliturl(data['url']) 341 if url_obj.scheme in WebService.SUPPORTED_URL_SCHEMAS: 342 data['_url_obj'] = url_obj 343 else: 344 data['url'] = None 345 346 for line in content: 347 for k in ("UPDATE_EAPI", "REPO_EAPI"): 348 if line.startswith(k + "="): 349 try: 350 data[k.lower()] = int(line.split("=", 1)[-1]) 351 except (IndexError, ValueError): 352 pass 353 354 return data
355
356 - def _set_timeout(self, secs):
357 """ 358 Override default timeout setting a new one (in seconds). 359 """ 360 self._default_timeout_secs = float(secs)
361
362 - def _set_transfer_callback(self, callback):
363 """ 364 Set a transfer progress callback function. 365 366 @param transfer_callback: this callback function can be used to 367 show a progress status to user, if passed, it must be a function 368 accepting 3 input parameters: (int transfered, int total, 369 bool download). The last parameter is True, when progress is about 370 download, False if upload. If no transfer information is declared, 371 total might be -1. 372 @param transfer_callback: callable 373 """ 374 self._transfer_callback = callback
375 376 @property
377 - def _settings(self):
378 """ 379 Get SystemSettings instance 380 """ 381 if self.__settings is None: 382 self.__settings = SystemSettings() 383 return self.__settings
384 385 @property
386 - def _arch(self):
387 """ 388 Get currently running Entropy architecture 389 """ 390 return self._settings['repositories']['arch']
391 392 @property
393 - def _product(self):
394 """ 395 Get currently running Entropy product 396 """ 397 return self._settings['repositories']['product']
398 399 @property
400 - def _branch(self):
401 """ 402 Get currently running Entropy branch 403 """ 404 return self._settings['repositories']['branch']
405 406 @property
407 - def _authstore(self):
408 """ 409 Repository authentication configuration storage interface. 410 Makes possible to retrieve on-disk stored user credentials. 411 """ 412 if self.__auth_storage is None: 413 self.__auth_storage = AuthenticationStorage() 414 return self.__auth_storage
415 416 @property
417 - def _cacher(self):
418 if self.__cacher is None: 419 self.__cacher = EntropyCacher() 420 return self.__cacher
421
422 - def _generate_user_agent(self, function_name):
423 """ 424 Generate a standard (entropy services centric) HTTP User Agent. 425 """ 426 uname = os.uname() 427 user_agent = "Entropy.Services/%s (compatible; %s; %s: %s %s %s)" % ( 428 etpConst['entropyversion'], 429 "Entropy", 430 function_name, 431 uname[0], 432 uname[4], 433 uname[2], 434 ) 435 return user_agent
436
437 - def _encode_multipart_form(self, params, file_params, boundary):
438 """ 439 Encode parameters and files into a valid HTTP multipart form data. 440 NOTE: this method loads the whole file in RAM, HTTP post doesn't work 441 well for big files anyway. 442 """ 443 def _cast_to_str(value): 444 if value is None: 445 return const_convert_to_rawstring("") 446 elif isinstance(value, (int, float, long)): 447 return const_convert_to_rawstring(value) 448 elif isinstance(value, (list, tuple)): 449 return repr(value) 450 return value
451 452 tmp_fd, tmp_path = const_mkstemp(prefix="_encode_multipart_form") 453 tmp_f = os.fdopen(tmp_fd, "ab+") 454 tmp_f.truncate(0) 455 crlf = '\r\n' 456 for key, value in params.items(): 457 tmp_f.write("--" + boundary + crlf) 458 tmp_f.write("Content-Disposition: form-data; name=\"%s\"" % ( 459 key,)) 460 tmp_f.write(crlf + crlf + _cast_to_str(value) + crlf) 461 for key, (f_name, f_obj) in file_params.items(): 462 tmp_f.write("--" + boundary + crlf) 463 tmp_f.write( 464 "Content-Disposition: form-data; name=\"%s\"; filename=\"%s\"" % ( 465 key, f_name,)) 466 tmp_f.write(crlf) 467 tmp_f.write("Content-Type: application/octet-stream" + crlf) 468 tmp_f.write("Content-Transfer-Encoding: binary" + crlf + crlf) 469 f_obj.seek(0) 470 while True: 471 chunk = f_obj.read(65536) 472 if not chunk: 473 break 474 tmp_f.write(chunk) 475 tmp_f.write(crlf) 476 477 tmp_f.write("--" + boundary + "--" + crlf + crlf) 478 tmp_f.flush() 479 return tmp_f, tmp_path
480
481 - def _generic_post_handler(self, function_name, params, file_params, 482 timeout):
483 """ 484 Given a function name and the request data (dict format), do the actual 485 HTTP request and return the response object to caller. 486 WARNING: params and file_params dict keys must be ASCII string only. 487 488 @param function_name: name of the function that called this method 489 @type function_name: string 490 @param params: POST parameters 491 @type params: dict 492 @param file_params: mapping composed by file names as key and tuple 493 composed by (file_name, file object) as values 494 @type file_params: dict 495 @param timeout: socket timeout 496 @type timeout: float 497 @return: tuple composed by the server response string or None 498 (in case of empty response) and the HTTPResponse object (useful 499 for checking response status) 500 @rtype: tuple 501 """ 502 if timeout is None: 503 timeout = self._default_timeout_secs 504 multipart_boundary = "---entropy.services,boundary---" 505 request_path = self._request_path.rstrip("/") + "/" + function_name 506 const_debug_write(__name__, 507 "WebService _generic_post_handler, calling: %s at %s -- %s," 508 " tx_callback: %s, timeout: %s" % (self._request_host, request_path, 509 params, self._transfer_callback, timeout,)) 510 connection = None 511 try: 512 if self._request_protocol == "http": 513 connection = httplib.HTTPConnection(self._request_host, 514 timeout = timeout) 515 elif self._request_protocol == "https": 516 ssl_context = None 517 if hasattr(ssl, 'create_default_context'): 518 ssl_context = ssl.create_default_context( 519 purpose = ssl.Purpose.CLIENT_AUTH) 520 connection = httplib.HTTPSConnection( 521 self._request_host, timeout = timeout, context = ssl_context) 522 else: 523 raise WebService.RequestError("invalid request protocol", 524 method = function_name) 525 526 headers = { 527 "Accept": "text/plain", 528 "User-Agent": self._generate_user_agent(function_name), 529 } 530 531 if file_params is None: 532 file_params = {} 533 # autodetect file parameters in params 534 for k in list(params.keys()): 535 if isinstance(params[k], (tuple, list)) \ 536 and (len(params[k]) == 2): 537 f_name, f_obj = params[k] 538 if isinstance(f_obj, file): 539 file_params[k] = params[k] 540 del params[k] 541 elif const_isunicode(params[k]): 542 # convert to raw string 543 params[k] = const_convert_to_rawstring(params[k], 544 from_enctype = "utf-8") 545 elif not const_isstring(params[k]): 546 # invalid ? 547 if params[k] is None: 548 # will be converted to "" 549 continue 550 int_types = const_get_int() 551 supported_types = (float, list, tuple) + int_types 552 if not isinstance(params[k], supported_types): 553 raise WebService.UnsupportedParameters( 554 "%s is unsupported type %s" % (k, type(params[k]))) 555 list_types = (list, tuple) 556 if isinstance(params[k], list_types): 557 # not supporting nested lists 558 non_str = [x for x in params[k] if not \ 559 const_isstring(x)] 560 if non_str: 561 raise WebService.UnsupportedParameters( 562 "%s is unsupported type %s" % (k, 563 type(params[k]))) 564 565 body = None 566 if not file_params: 567 headers["Content-Type"] = "application/x-www-form-urlencoded" 568 encoded_params = urllib_parse.urlencode(params) 569 data_size = len(encoded_params) 570 if self._transfer_callback is not None: 571 self._transfer_callback(0, data_size, False) 572 573 if data_size < 65536: 574 try: 575 connection.request("POST", request_path, encoded_params, 576 headers) 577 except socket.error as err: 578 raise WebService.RequestError(err, 579 method = function_name) 580 else: 581 try: 582 connection.request("POST", request_path, None, headers) 583 except socket.error as err: 584 raise WebService.RequestError(err, 585 method = function_name) 586 sio = StringIO(encoded_params) 587 data_size = len(encoded_params) 588 while True: 589 chunk = sio.read(65535) 590 if not chunk: 591 break 592 try: 593 connection.send(chunk) 594 except socket.error as err: 595 raise WebService.RequestError(err, 596 method = function_name) 597 if self._transfer_callback is not None: 598 self._transfer_callback(sio.tell(), 599 data_size, False) 600 # for both ways, send a signal through the callback 601 if self._transfer_callback is not None: 602 self._transfer_callback(data_size, data_size, False) 603 604 else: 605 headers["Content-Type"] = "multipart/form-data; boundary=" + \ 606 multipart_boundary 607 body_file, body_fpath = self._encode_multipart_form(params, 608 file_params, multipart_boundary) 609 try: 610 data_size = body_file.tell() 611 headers["Content-Length"] = str(data_size) 612 body_file.seek(0) 613 if self._transfer_callback is not None: 614 self._transfer_callback(0, data_size, False) 615 616 try: 617 connection.request("POST", request_path, None, headers) 618 except socket.error as err: 619 raise WebService.RequestError(err, 620 method = function_name) 621 while True: 622 chunk = body_file.read(65535) 623 if not chunk: 624 break 625 try: 626 connection.send(chunk) 627 except socket.error as err: 628 raise WebService.RequestError(err, 629 method = function_name) 630 if self._transfer_callback is not None: 631 self._transfer_callback(body_file.tell(), 632 data_size, False) 633 if self._transfer_callback is not None: 634 self._transfer_callback(data_size, data_size, False) 635 finally: 636 body_file.close() 637 os.remove(body_fpath) 638 639 try: 640 response = connection.getresponse() 641 except socket.error as err: 642 raise WebService.RequestError(err, 643 method = function_name) 644 const_debug_write(__name__, "WebService.%s(%s), " 645 "response header: %s" % ( 646 function_name, params, response.getheaders(),)) 647 total_length = response.getheader("Content-Length", "-1") 648 try: 649 total_length = int(total_length) 650 except ValueError: 651 total_length = -1 652 outcome = const_convert_to_rawstring("") 653 current_len = 0 654 if self._transfer_callback is not None: 655 self._transfer_callback(current_len, total_length, True) 656 while True: 657 try: 658 chunk = response.read(65536) 659 except socket.error as err: 660 raise WebService.RequestError(err, 661 method = function_name) 662 if not chunk: 663 break 664 outcome += chunk 665 current_len += len(chunk) 666 if self._transfer_callback is not None: 667 self._transfer_callback(current_len, total_length, True) 668 669 if self._transfer_callback is not None: 670 self._transfer_callback(total_length, total_length, True) 671 672 if const_is_python3(): 673 outcome = const_convert_to_unicode(outcome) 674 if not outcome: 675 return None, response 676 return outcome, response 677 678 except httplib.HTTPException as err: 679 raise WebService.RequestError(err, 680 method = function_name) 681 finally: 682 if connection is not None: 683 connection.close()
684
685 - def _setup_credentials(self, request_params):
686 """ 687 This method is automatically called by public API functions to setup 688 credentials data if available, otherwise user interaction will be 689 triggered by raising WebService.AuthenticationRequired 690 """ 691 creds = self._authstore.get(self._repository_id) 692 if creds is None: 693 raise WebService.AuthenticationRequired(self._repository_id) 694 username, password = creds 695 request_params['username'], request_params['password'] = \ 696 username, password
697
698 - def _setup_generic_params(self, request_params):
699 """ 700 This methods adds some generic parameters to the HTTP request metadata. 701 Any parameter added by this method is prefixed with __, to avoid 702 name collisions. 703 """ 704 request_params["__repository_id__"] = self._repository_id 705 request_params["__version__"] = etpConst['entropyversion']
706
707 - def enable_cache_aging(self, days):
708 """ 709 Turn on on-disk cache aging support. If cache is older than given 710 days, it will be removed and considered invalid. 711 """ 712 self._cache_aging_days = int(days)
713
714 - def add_credentials(self, username, password):
715 """ 716 Add credentials for this repository and store the information into 717 an user-protected location. 718 """ 719 self.__credentials_validated = False 720 self._authstore.add(self._repository_id, username, password) 721 self._authstore.save()
722
723 - def validate_credentials(self):
724 """ 725 Validate currently stored credentials (if available) against the 726 remote service. If credentials are not available, 727 WebService.AuthenticationRequired is raised. 728 If credentials are not valid, WebService.AuthenticationFailed is 729 raised. 730 731 @raise WebService.AuthenticationRequired: if credentials are not 732 available 733 @raise WebService.AuthenticationFailed: if credentials are not valid 734 """ 735 if not self.credentials_available(): 736 raise WebService.AuthenticationRequired("credentials not available") 737 if not self.__credentials_validated: 738 # this will raise WebService.AuthenticationFailed if credentials 739 # are invalid 740 self._method_getter("validate_credentials", {}, cache = False, 741 require_credentials = True) 742 self.__credentials_validated = True
743
744 - def credentials_available(self):
745 """ 746 Return whether credentials are stored locally or not. 747 Please note that credentials can be stored properly but considered 748 invalid remotely. 749 750 @return: True, if credentials are available 751 @rtype: bool 752 """ 753 return self._authstore.get(self._repository_id) is not None
754
755 - def get_credentials(self):
756 """ 757 Return the username string stored in the authentication storage, if any. 758 Otherwise return None. 759 760 @return: the username string stored in the authentication storage 761 @rtype: string or None 762 """ 763 creds = self._authstore.get(self._repository_id) 764 if creds is not None: 765 username, _pass = creds 766 return username
767
768 - def remove_credentials(self):
769 """ 770 Remove any credential bound to the repository from on-disk storage. 771 772 @return: True, if credentials existed and got removed 773 @rtype: bool 774 """ 775 self.__credentials_validated = False 776 res = self._authstore.remove(self._repository_id) 777 self._authstore.save() 778 return res
779 780 CACHE_DIR = os.path.join(etpConst['entropyworkdir'], "websrv_cache") 781
782 - def _get_cache_key(self, method, params):
783 """ 784 Return on disk cache file name as key, given a method name and its 785 parameters. 786 """ 787 sorted_data = [(x, params[x]) for x in sorted(params.keys())] 788 hash_str = repr(sorted_data) + ", " + self._request_url 789 if const_is_python3(): 790 hash_str = hash_str.encode("utf-8") 791 sha = hashlib.sha1() 792 sha.update(hash_str) 793 return method + "_" + sha.hexdigest()
794
795 - def _get_cached(self, cache_key):
796 """ 797 Return an on-disk cached object for given cache key. 798 """ 799 with self._cache_dir_lock: 800 return self._cacher.pop( 801 cache_key, cache_dir = WebService.CACHE_DIR, 802 aging_days = self._cache_aging_days)
803
804 - def _set_cached(self, cache_key, data):
805 """ 806 Save a cache item to disk. 807 """ 808 with self._cache_dir_lock: 809 try: 810 return self._cacher.save(cache_key, data, 811 cache_dir = WebService.CACHE_DIR) 812 except IOError as err: 813 # IOError is raised when cache cannot be written to disk 814 if const_debug_enabled(): 815 const_debug_write(__name__, 816 "WebService._set_cached(%s) = cache store error: %s" % ( 817 cache_key, repr(err),))
818
819 - def _drop_cached(self, method):
820 """ 821 Drop all on-disk cache for given method. 822 """ 823 with self._cache_dir_lock: 824 cache_dir = WebService.CACHE_DIR 825 for currentdir, subdirs, files in os.walk(cache_dir): 826 hostile_files = [os.path.join(currentdir, x) for x in \ 827 files if x.startswith(method + "_")] 828 for path in hostile_files: 829 try: 830 os.remove(path) 831 except OSError as err: 832 # avoid race conditions 833 if err.errno != errno.ENOENT: 834 raise
835
836 - def _method_cached(self, func_name, params, cache_key = None):
837 """ 838 Try to fetch on-disk cached object and return it. If error or not 839 found, None is returned. 840 """ 841 # setup generic request parameters 842 self._setup_generic_params(params) 843 844 if cache_key is None: 845 cache_key = self._get_cache_key(func_name, params) 846 return self._get_cached(cache_key)
847
848 - def _method_getter(self, func_name, params, cache = True, 849 cached = False, require_credentials = False, file_params = None, 850 timeout = None):
851 """ 852 Given a function name and request parameters, do all the duties required 853 to get a response from the Web Service. This method raises several 854 exceptions, that have to be advertised on public methods as well. 855 856 @param func_name: API function name 857 @type func_name: string 858 @param params: dictionary object that will be converted into a JSON 859 request string 860 @type params: dict 861 @keyword cache: True means use on-disk cache if available? 862 @type cache: bool 863 @keyword cached: if True, it will only use the on-disk cached call 864 result and raise WebService.CacheMiss if not found. 865 @type cached: bool 866 @keyword require_credentials: True means that credentials will be added 867 to the request, if credentials are not available in the local 868 authentication storage, WebService.AuthenticationRequired is 869 raised 870 @type require_credentials: bool 871 @param file_params: mapping composed by file names as key and tuple 872 composed by (file_name, file object) as values 873 @type file_params: dict 874 @param timeout: provide specific socket timeout 875 @type timeout: float 876 @return: the JSON response (dict format) 877 @rtype: dict 878 @raise WebService.UnsupportedParameters: if input parameters are invalid 879 @raise WebService.RequestError: if request cannot be satisfied 880 @raise WebService.MethodNotAvailable: if API method is not available 881 remotely and an error occurred (error code passed as exception 882 argument) 883 @raise WebService.AuthenticationRequired: if require_credentials is True 884 and credentials are required. 885 @raise WebService.AuthenticationFailed: if credentials are not valid 886 @raise WebService.MalformedResponse: if JSON response cannot be 887 converted back to dict. 888 @raise WebService.UnsupportedAPILevel: if client API and Web Service 889 API do not match 890 @raise WebService.MethodResponseError; if method execution failed 891 @raise WebService.CacheMiss: if cached=True and cached object is not 892 available 893 """ 894 cache_key = self._get_cache_key(func_name, params) 895 if cache or cached: 896 # this does call: _setup_generic_params() 897 obj = self._method_cached(func_name, params, cache_key = cache_key) 898 if (obj is None) and cached: 899 if const_debug_enabled(): 900 const_debug_write(__name__, 901 "WebService.%s(%s) = cache miss: %s" % ( 902 func_name, params, cache_key,)) 903 raise WebService.CacheMiss( 904 WebService.WEB_SERVICE_NOT_FOUND_CODE, method = func_name) 905 if obj is not None: 906 if const_debug_enabled(): 907 const_debug_write(__name__, 908 "WebService.%s(%s) = CACHED!" % ( 909 func_name, params,)) 910 return obj 911 if const_debug_enabled(): 912 const_debug_write(__name__, "WebService.%s(%s) = NOT cached" % ( 913 func_name, params,)) 914 else: 915 self._setup_generic_params(params) 916 917 if require_credentials: 918 # this can raise AuthenticationRequired 919 self._setup_credentials(params) 920 921 obj = None 922 try: 923 json_response, response = self._generic_post_handler(func_name, 924 params, file_params, timeout) 925 926 http_status = response.status 927 if http_status not in (httplib.OK,): 928 raise WebService.MethodNotAvailable(http_status, 929 method = func_name) 930 931 # try to convert the JSON response 932 try: 933 data = json.loads(json_response) 934 except (ValueError, TypeError) as err: 935 raise WebService.MalformedResponse(err, 936 method = func_name) 937 938 # check API 939 if data.get("api_rev") != WebService.SUPPORTED_API_LEVEL: 940 raise WebService.UnsupportedAPILevel(data['api_rev'], 941 method = func_name, message = data.get("message")) 942 943 code = data.get("code", -1) 944 if code == WebService.WEB_SERVICE_INVALID_CREDENTIALS_CODE: 945 # invalid credentials, ask again login data 946 raise WebService.AuthenticationFailed(code, 947 method = func_name, message = data.get("message")) 948 if code != WebService.WEB_SERVICE_RESPONSE_CODE_OK: 949 raise WebService.MethodResponseError(code, 950 method = func_name, message = data.get("message")) 951 952 if "r" not in data: 953 raise WebService.MalformedResponse("r not found", 954 method = func_name, message = data.get("message")) 955 obj = data["r"] 956 957 if const_debug_enabled(): 958 const_debug_write(__name__, "WebService.%s(%s) = fetched!" % ( 959 func_name, params,)) 960 return obj 961 962 finally: 963 if obj is not None: 964 # store cache 965 self._set_cached(cache_key, obj)
966
967 - def service_available(self, cache = True, cached = False):
968 """ 969 Return whether the Web Service is correctly able to answer our requests. 970 971 @keyword cache: True means use on-disk cache if available? 972 @type cache: bool 973 @keyword cached: if True, it will only use the on-disk cached call 974 result and raise WebService.CacheMiss if not found. 975 @type cached: bool 976 @return: True, if service is available 977 @rtype: bool 978 979 @raise WebService.UnsupportedParameters: if input parameters are invalid 980 @raise WebService.RequestError: if request cannot be satisfied 981 @raise WebService.MethodNotAvailable: if API method is not available 982 remotely and an error occurred (error code passed as exception 983 argument) 984 @raise WebService.AuthenticationRequired: if require_credentials is True 985 and credentials are required. 986 @raise WebService.AuthenticationFailed: if credentials are not valid 987 @raise WebService.MalformedResponse: if JSON response cannot be 988 converted back to dict. 989 @raise WebService.UnsupportedAPILevel: if client API and Web Service 990 API do not match 991 @raise WebService.MethodResponseError; if method execution failed 992 @raise WebService.CacheMiss: if cached=True and cached object is not 993 available 994 """ 995 params = locals().copy() 996 params.pop("self") 997 params.pop("cache") 998 return self._method_getter("service_available", params, cache = cache, 999 cached = cached, require_credentials = False)
1000
1001 - def data_send_available(self):
1002 """ 1003 Return whether data send is correctly working. A temporary file with 1004 random content is sent to the service, that would need to calculate 1005 its md5 hash. For security reason, data will be accepted remotely if, 1006 and only if its size is < 256 bytes. 1007 """ 1008 md5 = hashlib.md5() 1009 test_str = const_convert_to_rawstring("") 1010 for x in range(256): 1011 test_str += chr(x) 1012 md5.update(test_str) 1013 expected_hash = md5.hexdigest() 1014 func_name = "data_send_available" 1015 1016 tmp_fd, tmp_path = const_mkstemp(prefix="data_send_available") 1017 try: 1018 with os.fdopen(tmp_fd, "ab+") as tmp_f: 1019 tmp_f.write(test_str) 1020 tmp_f.seek(0) 1021 params = { 1022 "test_param": "hello", 1023 } 1024 file_params = { 1025 "test_file": ("test_file.txt", tmp_f), 1026 } 1027 remote_hash = self._method_getter(func_name, params, 1028 cache = False, require_credentials = False, 1029 file_params = file_params) 1030 finally: 1031 os.remove(tmp_path) 1032 1033 const_debug_write(__name__, 1034 "WebService.%s, expected: %s, got: %s" % ( 1035 func_name, repr(expected_hash), repr(remote_hash),)) 1036 return expected_hash == remote_hash
1037
1038 1039 -class AuthenticationStorage(Singleton):
1040 """ 1041 Entropy Web Service authentication credentials storage class. 1042 """ 1043 1044 _AUTH_FILE = ".entropy/id_entropy" 1045
1046 - def init_singleton(self):
1047 1048 self.__dump_lock = threading.Lock() 1049 # not loaded, load at very last moment 1050 self.__store = None
1051
1052 - def _get_authfile(self):
1053 """ 1054 Try to get the auth file. If it fails, return None. 1055 """ 1056 # setup auth file path 1057 home = os.getenv("HOME") 1058 auth_file = None 1059 if home is not None: 1060 if const_dir_writable(home): 1061 auth_file = os.path.join(home, 1062 AuthenticationStorage._AUTH_FILE) 1063 auth_dir = os.path.dirname(auth_file) 1064 if not os.path.isdir(auth_dir): 1065 try: 1066 os.makedirs(auth_dir, 0o700) 1067 const_setup_file(auth_dir, etpConst['entropygid'], 1068 0o700) 1069 except (OSError, IOError): 1070 # ouch, no permissions 1071 auth_file = None 1072 return auth_file
1073 1074 @property
1075 - def _authstore(self):
1076 """ 1077 Authentication data object automatically loaded from disk if needed. 1078 """ 1079 if self.__store is None: 1080 auth_file = self._get_authfile() 1081 store = {} 1082 if auth_file is not None: 1083 store = entropy.dump.loadobj(auth_file, complete_path = True) 1084 if store is None: 1085 store = {} 1086 elif not isinstance(store, dict): 1087 store = {} 1088 self.__store = store 1089 return self.__store
1090
1091 - def save(self):
1092 """ 1093 Save currently loaded authentication configuration to disk. 1094 1095 @return: True, if save was effectively run 1096 @rtype: bool 1097 """ 1098 with self.__dump_lock: 1099 auth_file = self._get_authfile() 1100 if auth_file is not None: 1101 entropy.dump.dumpobj(auth_file, self._authstore, 1102 complete_path = True, custom_permissions = 0o600) 1103 # make sure 1104 if auth_file is not None: 1105 try: 1106 const_setup_file(auth_file, etpConst['entropygid'], 1107 0o600) 1108 return True 1109 except (OSError, IOError): 1110 return False
1111
1112 - def add(self, repository_id, username, password):
1113 """ 1114 Add authentication credentials to Authentication configuration. 1115 1116 @param repository_id: repository identifier 1117 @type repository_id: string 1118 @param username: the username 1119 @type username: string 1120 @param password: the password 1121 @type password: string 1122 """ 1123 self._authstore[repository_id] = { 1124 'username': username, 1125 'password': password, 1126 }
1127
1128 - def remove(self, repository_id):
1129 """ 1130 Remove any credential for given repository identifier. 1131 1132 @param repository_id: repository identifier 1133 @type repository_id: string 1134 @return: True, if removal went fine (if there was something to remove) 1135 @rtype: bool 1136 """ 1137 try: 1138 del self._authstore[repository_id] 1139 return True 1140 except KeyError: 1141 return False
1142
1143 - def get(self, repository_id):
1144 """ 1145 Get authentication credentials for given repository identifier. 1146 1147 @return: tuple composed by username, password, or None, if credentials 1148 are not found 1149 @rtype: tuple or None 1150 """ 1151 data = self._authstore.get(repository_id) 1152 if data is None: 1153 return None 1154 return data['username'], data['password']
1155