Package entropy :: Package services :: Module client

Source Code for Module entropy.services.client

   1  # -*- coding: utf-8 -*- 
   2  """ 
   3   
   4      @author: Fabio Erculiani <[email protected]> 
   5      @contact: [email protected] 
   6      @copyright: Fabio Erculiani 
   7      @license: GPL-2 
   8   
   9      B{Entropy Base Repository Web Services client interface}. 
  10   
  11  """ 
  12  __all__ = ["WebServiceFactory", "WebService"] 
  13   
  14  import sys 
  15  import os 
  16  import errno 
  17  import json 
  18  import threading 
  19  import hashlib 
  20  import ssl 
  21  import socket 
  22   
  23  from entropy.const import const_is_python3, const_convert_to_rawstring, \ 
  24      const_get_int, const_mkstemp, const_dir_writable, const_get_int 
  25   
  26  if const_is_python3(): 
  27      import http.client as httplib 
  28      from io import StringIO 
  29      import urllib.parse as urllib_parse 
  30  else: 
  31      import httplib 
  32      from cStringIO import StringIO 
  33      import urllib as urllib_parse 
  34   
  35  import entropy.dump 
  36  from entropy.core import Singleton 
  37  from entropy.cache import EntropyCacher 
  38  from entropy.const import const_debug_write, const_setup_file, etpConst, \ 
  39      const_convert_to_rawstring, const_isunicode, const_isstring, \ 
  40      const_convert_to_unicode, const_isstring, const_debug_enabled, \ 
  41      const_isfileobj 
  42  from entropy.core.settings.base import SystemSettings 
  43  from entropy.exceptions import EntropyException 
  44  import entropy.tools 
  45  import entropy.dep 
46 47 48 -class WebServiceFactory(object):
49 """ 50 Base Entropy Repository Web Services Factory. Generates 51 WebService objects that can be used to communicate with the established 52 web service. 53 This is a base class and subclasses should be preferred (example: 54 entropy.client.services.interfaces.ClientWebServiceFactory) 55 """
56 - class InvalidWebServiceFactory(EntropyException):
57 """ 58 Raised when an invalid WebService based class is passed. 59 """
60
61 - def __init__(self, web_service_class, entropy_client):
62 """ 63 WebServiceFactory constructor. 64 65 @param entropy_client: Entropy Client interface 66 @type entropy_client: entropy.client.interfaces.client.Client 67 """ 68 object.__init__(self) 69 if not issubclass(web_service_class, WebService): 70 raise WebServiceFactory.InvalidWebServiceFactory( 71 "invalid web_service_class") 72 self._entropy = entropy_client 73 self._service_class = web_service_class
74
75 - def new(self, repository_id):
76 """ 77 Return a new WebService object for given repository identifier. 78 79 @param repository_id: repository identifier 80 @rtype repository_id: string 81 @raise WebService.UnsupportedService: if web service is 82 explicitly unsupported by repository 83 """ 84 return self._service_class(self._entropy, repository_id)
85
86 87 -class WebService(object):
88 """ 89 This is the Entropy Repository Web Services that proxies requests over 90 an Web Services answering to HTTP POST requests 91 (either over HTTP or over HTTPS) in the following form: 92 93 Given that repositories must ship with a file (in their repository 94 meta file "packages.db.meta", coming from the server-side repository 95 directory) called "packages.db.webservices" 96 (etpConst['etpdatabasewebservicesfile']) containing the HTTP POST base 97 URL (example: http://packages.sabayon.org/api). 98 The POST url is composed as follows: 99 <base_url>/<method name> 100 Function arguments are then appended in JSON format, so that float, int, 101 strings and lists are correctly represented. 102 So, for example, if WebService exposes a method with the following 103 signature (with the base URL in example above): 104 105 float get_vote(string package_name) 106 107 The URL referenced will be: 108 109 http://packages.sabayon.org/api/get_vote 110 111 And the JSON dictionary will contain a key called "package_name" with 112 package_name value. 113 For methods requiring authentication, the JSON object will contain 114 "username" and "password" fields (clear text, so make sure to use HTTPS). 115 116 The Response depends on each specific method and it is given in JSON format 117 too, that is afterwards interpreted by the caller function, that will 118 always return the expected data format (see respective API documentation). 119 For more information about how to implement the Web Service, please see 120 the packages.git Sabayon repository, which contains a Pylons MVC web app. 121 In general, every JSON response must provide a 'code' field, representing 122 an HTTP-response alike return code (200 is ok, 500 is server error, 400 is 123 bad request, etc) and a 'message' field, containing the error message (if 124 no error, 'message' is usually empty). The RPC result is put inside the 125 'r' field. 126 127 This is a base class, and you should really implement a subclass providing 128 your own API methods, and use _method_getter(). 129 """ 130 131 # Supported communcation protocols 132 SUPPORTED_URL_SCHEMAS = ("http", "https") 133 134 # package icon metadata identifier 135 PKG_ICON_IDENTIFIER = "__icon__" 136 137 # Currently supported Web Service API level 138 # an API level defines a set of available remote calls and their data 139 # structure 140 SUPPORTED_API_LEVEL = 1 141 142 # Default common Web Service responses, please use these when 143 # implementing your web service 144 WEB_SERVICE_RESPONSE_CODE_OK = 200 145 WEB_SERVICE_INVALID_CREDENTIALS_CODE = 450 146 WEB_SERVICE_INVALID_REQUEST_CODE = 400 147 WEB_SERVICE_NOT_FOUND_CODE = 404 148 WEB_SERVICE_RESPONSE_ERROR_CODE = 503 149 150
151 - class WebServiceException(EntropyException):
152 """ 153 Base WebService exception class. 154 """
155 - def __init__(self, value, method = None, message = None):
156 self.value = value 157 self.method = method 158 self.message = message 159 Exception.__init__(self)
160
161 - def __get_method(self):
162 if self.method is None: 163 method = const_convert_to_unicode("") 164 else: 165 method = const_convert_to_unicode(self.method) 166 return method
167
168 - def __get_message(self):
169 if self.message is None: 170 message = const_convert_to_unicode("") 171 else: 172 message = const_convert_to_unicode(self.message) 173 return message
174
175 - def __unicode__(self):
176 method = self.__get_method() 177 message = self.__get_message() 178 if const_isstring(self.value): 179 return const_convert_to_unicode(method + " " + self.value) \ 180 + ", " + message 181 return const_convert_to_unicode(method + " " + repr(self.value)) \ 182 + ", " + message
183
184 - def __str__(self):
185 method = self.__get_method() 186 message = self.__get_message() 187 if const_isstring(self.value): 188 return method + " " + self.value + ", " + message 189 return method + " " + repr(self.value) + ", " + message
190
191 - class UnsupportedService(WebServiceException):
192 """ 193 Raised when Repository doesn't seem to support any Web Service 194 feature. 195 """
196
197 - class UnsupportedParameters(WebServiceException):
198 """ 199 Raised when input parameters cannot be converted to JSON. 200 Probably due to invalid input data. 201 """
202
203 - class RequestError(WebServiceException):
204 """ 205 If the request cannot be satisfied by the remote web service. 206 """
207
208 - class AuthenticationRequired(WebServiceException):
209 """ 210 When a method requiring valid user credentials is called without 211 being logged in. 212 """
213
214 - class AuthenticationFailed(WebServiceException):
215 """ 216 When credentials are stored locally but don't seem to work against 217 the Web Service. 218 """
219
220 - class MethodNotAvailable(WebServiceException):
221 """ 222 When calling a remote method that is not available. 223 """
224
225 - class MalformedResponse(WebServiceException):
226 """ 227 If JSON response cannot be converted back to dict. 228 """
229
230 - class UnsupportedAPILevel(WebServiceException):
231 """ 232 If this client and the Web Service expose a different API level. 233 """
234
235 - class MethodResponseError(WebServiceException):
236 """ 237 If the request has been accepted, but its computation stopped for 238 some reason. The encapsulated data contains the error code. 239 """
240
241 - class CacheMiss(WebServiceException):
242 """ 243 If the request is not available in the on-disk cache. 244 """
245
246 - def __init__(self, entropy_client, repository_id):
247 """ 248 WebService constructor. 249 NOTE: This base class must NOT use any Entropy Client specific method 250 and MUST rely on what is provided by it's parent class TextInterface. 251 252 @param entropy_client: Entropy Client interface 253 @type entropy_client: entropy.client.interfaces.client.Client 254 @param repository_id: repository identifier 255 @rtype repository_id: string 256 """ 257 self._cache_dir_lock = threading.RLock() 258 self._transfer_callback = None 259 self._entropy = entropy_client 260 self._repository_id = repository_id 261 self.__auth_storage = None 262 self.__settings = None 263 self.__cacher = None 264 self._default_timeout_secs = 10.0 265 self.__credentials_validated = False 266 # if this is set, cache will be considered invalid if older than 267 self._cache_aging_days = None 268 269 config = self.config(repository_id) 270 if config is None: 271 raise WebService.UnsupportedService("unsupported service [1]") 272 273 remote_url = config['url'] 274 if remote_url is None: 275 raise WebService.UnsupportedService("unsupported service [2]") 276 url_obj = config['_url_obj'] 277 278 self._request_url = remote_url 279 self._request_protocol = url_obj.scheme 280 self._request_host = url_obj.netloc 281 self._request_path = url_obj.path 282 self._config = config 283 284 const_debug_write(__name__, "WebService loaded, url: %s" % ( 285 self._request_url,))
286 287 @classmethod
288 - def config(cls, repository_id):
289 """ 290 Return the WebService configuration for the given repository. 291 The object returned is a dictionary containing the following 292 items: 293 - url: the Entropy WebService base URL (or None, if not supported) 294 - update_eapi: the maximum supported EAPI for repository updates. 295 - repo_eapi: the maximum supported EAPI for User Generate Content. 296 297 @param repository_id: repository identifier 298 @type repository_id: string 299 """ 300 settings = SystemSettings() 301 _repository_data = settings['repositories']['available'].get( 302 repository_id) 303 if _repository_data is None: 304 const_debug_write(__name__, "WebService.config error: no repo") 305 return None 306 307 web_services_conf = _repository_data.get('webservices_config') 308 if web_services_conf is None: 309 const_debug_write(__name__, "WebService.config error: no metadata") 310 return None 311 312 data = { 313 'url': None, 314 '_url_obj': None, 315 'update_eapi': None, 316 'repo_eapi': None, 317 } 318 319 content = [] 320 try: 321 content += entropy.tools.generic_file_content_parser( 322 web_services_conf, encoding = etpConst['conf_encoding']) 323 except (OSError, IOError) as err: 324 const_debug_write(__name__, "WebService.config error: %s" % ( 325 err,)) 326 return None 327 328 if not content: 329 const_debug_write( 330 __name__, "WebService.config error: empty config") 331 return None 332 333 remote_url = content.pop(0) 334 if remote_url == "-": # as per specs 335 remote_url = None 336 elif not remote_url: 337 remote_url = None 338 data['url'] = remote_url 339 340 if data['url']: 341 url_obj = entropy.tools.spliturl(data['url']) 342 if url_obj.scheme in WebService.SUPPORTED_URL_SCHEMAS: 343 data['_url_obj'] = url_obj 344 else: 345 data['url'] = None 346 347 for line in content: 348 for k in ("UPDATE_EAPI", "REPO_EAPI"): 349 if line.startswith(k + "="): 350 try: 351 data[k.lower()] = int(line.split("=", 1)[-1]) 352 except (IndexError, ValueError): 353 pass 354 355 return data
356
357 - def _set_timeout(self, secs):
358 """ 359 Override default timeout setting a new one (in seconds). 360 """ 361 self._default_timeout_secs = float(secs)
362
363 - def _set_transfer_callback(self, callback):
364 """ 365 Set a transfer progress callback function. 366 367 @param transfer_callback: this callback function can be used to 368 show a progress status to user, if passed, it must be a function 369 accepting 3 input parameters: (int transfered, int total, 370 bool download). The last parameter is True, when progress is about 371 download, False if upload. If no transfer information is declared, 372 total might be -1. 373 @param transfer_callback: callable 374 """ 375 self._transfer_callback = callback
376 377 @property
378 - def _settings(self):
379 """ 380 Get SystemSettings instance 381 """ 382 if self.__settings is None: 383 self.__settings = SystemSettings() 384 return self.__settings
385 386 @property
387 - def _arch(self):
388 """ 389 Get currently running Entropy architecture 390 """ 391 return self._settings['repositories']['arch']
392 393 @property
394 - def _product(self):
395 """ 396 Get currently running Entropy product 397 """ 398 return self._settings['repositories']['product']
399 400 @property
401 - def _branch(self):
402 """ 403 Get currently running Entropy branch 404 """ 405 return self._settings['repositories']['branch']
406 407 @property
408 - def _authstore(self):
409 """ 410 Repository authentication configuration storage interface. 411 Makes possible to retrieve on-disk stored user credentials. 412 """ 413 if self.__auth_storage is None: 414 self.__auth_storage = AuthenticationStorage() 415 return self.__auth_storage
416 417 @property
418 - def _cacher(self):
419 if self.__cacher is None: 420 self.__cacher = EntropyCacher() 421 return self.__cacher
422
423 - def _generate_user_agent(self, function_name):
424 """ 425 Generate a standard (entropy services centric) HTTP User Agent. 426 """ 427 uname = os.uname() 428 user_agent = "Entropy.Services/%s (compatible; %s; %s: %s %s %s)" % ( 429 etpConst['entropyversion'], 430 "Entropy", 431 function_name, 432 uname[0], 433 uname[4], 434 uname[2], 435 ) 436 return user_agent
437
438 - def _encode_multipart_form(self, params, file_params, boundary):
439 """ 440 Encode parameters and files into a valid HTTP multipart form data. 441 NOTE: this method loads the whole file in RAM, HTTP post doesn't work 442 well for big files anyway. 443 """ 444 def _cast_to_str(value): 445 if value is None: 446 return const_convert_to_rawstring("") 447 elif isinstance(value, const_get_int() + (float,)): 448 return const_convert_to_rawstring(value) 449 elif isinstance(value, (list, tuple)): 450 return repr(value) 451 return value
452 453 tmp_fd, tmp_path = const_mkstemp(prefix="_encode_multipart_form") 454 tmp_f = os.fdopen(tmp_fd, "ab+") 455 tmp_f.truncate(0) 456 457 crlf = const_convert_to_rawstring('\r\n') 458 dashes = const_convert_to_rawstring("--") 459 raw_boundary = const_convert_to_rawstring(boundary) 460 for key, value in params.items(): 461 tmp_f.write(dashes + raw_boundary + crlf) 462 tmp_f.write( 463 const_convert_to_rawstring( 464 "Content-Disposition: form-data; name=\"%s\"" % (key,))) 465 tmp_f.write(crlf + crlf + _cast_to_str(value) + crlf) 466 for key, (f_name, f_obj) in file_params.items(): 467 tmp_f.write(dashes + raw_boundary + crlf) 468 tmp_f.write( 469 const_convert_to_rawstring( 470 "Content-Disposition: form-data; name=\"%s\"; filename=\"%s\"" % ( 471 key, f_name,))) 472 tmp_f.write(crlf) 473 tmp_f.write( 474 const_convert_to_rawstring( 475 "Content-Type: application/octet-stream") + crlf) 476 tmp_f.write( 477 const_convert_to_rawstring( 478 "Content-Transfer-Encoding: binary") + crlf + crlf) 479 f_obj.seek(0) 480 while True: 481 chunk = f_obj.read(65536) 482 if not chunk: 483 break 484 tmp_f.write(chunk) 485 tmp_f.write(crlf) 486 487 tmp_f.write(dashes + raw_boundary + dashes + crlf + crlf) 488 tmp_f.flush() 489 return tmp_f, tmp_path
490
491 - def _generic_post_handler(self, function_name, params, file_params, 492 timeout):
493 """ 494 Given a function name and the request data (dict format), do the actual 495 HTTP request and return the response object to caller. 496 WARNING: params and file_params dict keys must be ASCII string only. 497 498 @param function_name: name of the function that called this method 499 @type function_name: string 500 @param params: POST parameters 501 @type params: dict 502 @param file_params: mapping composed by file names as key and tuple 503 composed by (file_name, file object) as values 504 @type file_params: dict 505 @param timeout: socket timeout 506 @type timeout: float 507 @return: tuple composed by the server response string or None 508 (in case of empty response) and the HTTPResponse object (useful 509 for checking response status) 510 @rtype: tuple 511 """ 512 if timeout is None: 513 timeout = self._default_timeout_secs 514 multipart_boundary = "---entropy.services,boundary---" 515 request_path = self._request_path.rstrip("/") + "/" + function_name 516 const_debug_write(__name__, 517 "WebService _generic_post_handler, calling: %s at %s -- %s," 518 " tx_callback: %s, timeout: %s" % (self._request_host, request_path, 519 params, self._transfer_callback, timeout,)) 520 connection = None 521 try: 522 if self._request_protocol == "http": 523 connection = httplib.HTTPConnection(self._request_host, 524 timeout = timeout) 525 elif self._request_protocol == "https": 526 ssl_context = None 527 if hasattr(ssl, 'create_default_context'): 528 ssl_context = ssl.create_default_context( 529 purpose = ssl.Purpose.CLIENT_AUTH) 530 connection = httplib.HTTPSConnection( 531 self._request_host, timeout = timeout, context = ssl_context) 532 else: 533 raise WebService.RequestError("invalid request protocol", 534 method = function_name) 535 536 headers = { 537 "Accept": "text/plain", 538 "User-Agent": self._generate_user_agent(function_name), 539 } 540 541 if file_params is None: 542 file_params = {} 543 # autodetect file parameters in params 544 for k in list(params.keys()): 545 if isinstance(params[k], (tuple, list)) \ 546 and (len(params[k]) == 2): 547 f_name, f_obj = params[k] 548 if const_isfileobj(f_obj): 549 file_params[k] = params[k] 550 del params[k] 551 elif const_isunicode(params[k]): 552 # convert to raw string 553 params[k] = const_convert_to_rawstring(params[k], 554 from_enctype = "utf-8") 555 elif not const_isstring(params[k]): 556 # invalid ? 557 if params[k] is None: 558 # will be converted to "" 559 continue 560 int_types = const_get_int() 561 supported_types = (float, list, tuple) + int_types 562 if not isinstance(params[k], supported_types): 563 raise WebService.UnsupportedParameters( 564 "%s is unsupported type %s" % (k, type(params[k]))) 565 list_types = (list, tuple) 566 if isinstance(params[k], list_types): 567 # not supporting nested lists 568 non_str = [x for x in params[k] if not \ 569 const_isstring(x)] 570 if non_str: 571 raise WebService.UnsupportedParameters( 572 "%s is unsupported type %s" % (k, 573 type(params[k]))) 574 575 body = None 576 if not file_params: 577 headers["Content-Type"] = "application/x-www-form-urlencoded" 578 encoded_params = urllib_parse.urlencode(params) 579 data_size = len(encoded_params) 580 if self._transfer_callback is not None: 581 self._transfer_callback(0, data_size, False) 582 583 if data_size < 65536: 584 try: 585 connection.request("POST", request_path, encoded_params, 586 headers) 587 except socket.error as err: 588 raise WebService.RequestError(err, 589 method = function_name) 590 else: 591 try: 592 connection.request("POST", request_path, None, headers) 593 except socket.error as err: 594 raise WebService.RequestError(err, 595 method = function_name) 596 sio = StringIO(encoded_params) 597 data_size = len(encoded_params) 598 while True: 599 chunk = sio.read(65535) 600 if not chunk: 601 break 602 try: 603 connection.send(chunk) 604 except socket.error as err: 605 raise WebService.RequestError(err, 606 method = function_name) 607 if self._transfer_callback is not None: 608 self._transfer_callback(sio.tell(), 609 data_size, False) 610 # for both ways, send a signal through the callback 611 if self._transfer_callback is not None: 612 self._transfer_callback(data_size, data_size, False) 613 614 else: 615 headers["Content-Type"] = "multipart/form-data; boundary=" + \ 616 multipart_boundary 617 body_file, body_fpath = self._encode_multipart_form(params, 618 file_params, multipart_boundary) 619 try: 620 data_size = body_file.tell() 621 headers["Content-Length"] = str(data_size) 622 body_file.seek(0) 623 if self._transfer_callback is not None: 624 self._transfer_callback(0, data_size, False) 625 626 try: 627 connection.request("POST", request_path, None, headers) 628 except socket.error as err: 629 raise WebService.RequestError(err, 630 method = function_name) 631 while True: 632 chunk = body_file.read(65535) 633 if not chunk: 634 break 635 try: 636 connection.send(chunk) 637 except socket.error as err: 638 raise WebService.RequestError(err, 639 method = function_name) 640 if self._transfer_callback is not None: 641 self._transfer_callback(body_file.tell(), 642 data_size, False) 643 if self._transfer_callback is not None: 644 self._transfer_callback(data_size, data_size, False) 645 finally: 646 body_file.close() 647 os.remove(body_fpath) 648 649 try: 650 response = connection.getresponse() 651 except socket.error as err: 652 raise WebService.RequestError(err, 653 method = function_name) 654 const_debug_write(__name__, "WebService.%s(%s), " 655 "response header: %s" % ( 656 function_name, params, response.getheaders(),)) 657 total_length = response.getheader("Content-Length", "-1") 658 try: 659 total_length = int(total_length) 660 except ValueError: 661 total_length = -1 662 outcome = const_convert_to_rawstring("") 663 current_len = 0 664 if self._transfer_callback is not None: 665 self._transfer_callback(current_len, total_length, True) 666 while True: 667 try: 668 chunk = response.read(65536) 669 except socket.error as err: 670 raise WebService.RequestError(err, 671 method = function_name) 672 if not chunk: 673 break 674 outcome += chunk 675 current_len += len(chunk) 676 if self._transfer_callback is not None: 677 self._transfer_callback(current_len, total_length, True) 678 679 if self._transfer_callback is not None: 680 self._transfer_callback(total_length, total_length, True) 681 682 if const_is_python3(): 683 outcome = const_convert_to_unicode(outcome) 684 if not outcome: 685 return None, response 686 return outcome, response 687 688 except httplib.HTTPException as err: 689 raise WebService.RequestError(err, 690 method = function_name) 691 finally: 692 if connection is not None: 693 connection.close()
694
695 - def _setup_credentials(self, request_params):
696 """ 697 This method is automatically called by public API functions to setup 698 credentials data if available, otherwise user interaction will be 699 triggered by raising WebService.AuthenticationRequired 700 """ 701 creds = self._authstore.get(self._repository_id) 702 if creds is None: 703 raise WebService.AuthenticationRequired(self._repository_id) 704 username, password = creds 705 request_params['username'], request_params['password'] = \ 706 username, password
707
708 - def _setup_generic_params(self, request_params):
709 """ 710 This methods adds some generic parameters to the HTTP request metadata. 711 Any parameter added by this method is prefixed with __, to avoid 712 name collisions. 713 """ 714 request_params["__repository_id__"] = self._repository_id 715 request_params["__version__"] = etpConst['entropyversion']
716
717 - def enable_cache_aging(self, days):
718 """ 719 Turn on on-disk cache aging support. If cache is older than given 720 days, it will be removed and considered invalid. 721 """ 722 self._cache_aging_days = int(days)
723
724 - def add_credentials(self, username, password):
725 """ 726 Add credentials for this repository and store the information into 727 an user-protected location. 728 """ 729 self.__credentials_validated = False 730 self._authstore.add(self._repository_id, username, password) 731 self._authstore.save()
732
733 - def validate_credentials(self):
734 """ 735 Validate currently stored credentials (if available) against the 736 remote service. If credentials are not available, 737 WebService.AuthenticationRequired is raised. 738 If credentials are not valid, WebService.AuthenticationFailed is 739 raised. 740 741 @raise WebService.AuthenticationRequired: if credentials are not 742 available 743 @raise WebService.AuthenticationFailed: if credentials are not valid 744 """ 745 if not self.credentials_available(): 746 raise WebService.AuthenticationRequired("credentials not available") 747 if not self.__credentials_validated: 748 # this will raise WebService.AuthenticationFailed if credentials 749 # are invalid 750 self._method_getter("validate_credentials", {}, cache = False, 751 require_credentials = True) 752 self.__credentials_validated = True
753
754 - def credentials_available(self):
755 """ 756 Return whether credentials are stored locally or not. 757 Please note that credentials can be stored properly but considered 758 invalid remotely. 759 760 @return: True, if credentials are available 761 @rtype: bool 762 """ 763 return self._authstore.get(self._repository_id) is not None
764
765 - def get_credentials(self):
766 """ 767 Return the username string stored in the authentication storage, if any. 768 Otherwise return None. 769 770 @return: the username string stored in the authentication storage 771 @rtype: string or None 772 """ 773 creds = self._authstore.get(self._repository_id) 774 if creds is not None: 775 username, _pass = creds 776 return username
777
778 - def remove_credentials(self):
779 """ 780 Remove any credential bound to the repository from on-disk storage. 781 782 @return: True, if credentials existed and got removed 783 @rtype: bool 784 """ 785 self.__credentials_validated = False 786 res = self._authstore.remove(self._repository_id) 787 self._authstore.save() 788 return res
789 790 CACHE_DIR = os.path.join(etpConst['entropyworkdir'], "websrv_cache") 791
792 - def _get_cache_key(self, method, params):
793 """ 794 Return on disk cache file name as key, given a method name and its 795 parameters. 796 """ 797 sorted_data = [(x, params[x]) for x in sorted(params.keys())] 798 hash_str = repr(sorted_data) + ", " + self._request_url 799 if const_is_python3(): 800 hash_str = hash_str.encode("utf-8") 801 sha = hashlib.sha1() 802 sha.update(hash_str) 803 return method + "_" + sha.hexdigest()
804
805 - def _get_cached(self, cache_key):
806 """ 807 Return an on-disk cached object for given cache key. 808 """ 809 with self._cache_dir_lock: 810 return self._cacher.pop( 811 cache_key, cache_dir = WebService.CACHE_DIR, 812 aging_days = self._cache_aging_days)
813
814 - def _set_cached(self, cache_key, data):
815 """ 816 Save a cache item to disk. 817 """ 818 with self._cache_dir_lock: 819 try: 820 return self._cacher.save(cache_key, data, 821 cache_dir = WebService.CACHE_DIR) 822 except IOError as err: 823 # IOError is raised when cache cannot be written to disk 824 if const_debug_enabled(): 825 const_debug_write(__name__, 826 "WebService._set_cached(%s) = cache store error: %s" % ( 827 cache_key, repr(err),))
828
829 - def _drop_cached(self, method):
830 """ 831 Drop all on-disk cache for given method. 832 """ 833 with self._cache_dir_lock: 834 cache_dir = WebService.CACHE_DIR 835 for currentdir, subdirs, files in os.walk(cache_dir): 836 hostile_files = [os.path.join(currentdir, x) for x in \ 837 files if x.startswith(method + "_")] 838 for path in hostile_files: 839 try: 840 os.remove(path) 841 except OSError as err: 842 # avoid race conditions 843 if err.errno != errno.ENOENT: 844 raise
845
846 - def _method_cached(self, func_name, params, cache_key = None):
847 """ 848 Try to fetch on-disk cached object and return it. If error or not 849 found, None is returned. 850 """ 851 # setup generic request parameters 852 self._setup_generic_params(params) 853 854 if cache_key is None: 855 cache_key = self._get_cache_key(func_name, params) 856 return self._get_cached(cache_key)
857
858 - def _method_getter(self, func_name, params, cache = True, 859 cached = False, require_credentials = False, file_params = None, 860 timeout = None):
861 """ 862 Given a function name and request parameters, do all the duties required 863 to get a response from the Web Service. This method raises several 864 exceptions, that have to be advertised on public methods as well. 865 866 @param func_name: API function name 867 @type func_name: string 868 @param params: dictionary object that will be converted into a JSON 869 request string 870 @type params: dict 871 @keyword cache: True means use on-disk cache if available? 872 @type cache: bool 873 @keyword cached: if True, it will only use the on-disk cached call 874 result and raise WebService.CacheMiss if not found. 875 @type cached: bool 876 @keyword require_credentials: True means that credentials will be added 877 to the request, if credentials are not available in the local 878 authentication storage, WebService.AuthenticationRequired is 879 raised 880 @type require_credentials: bool 881 @param file_params: mapping composed by file names as key and tuple 882 composed by (file_name, file object) as values 883 @type file_params: dict 884 @param timeout: provide specific socket timeout 885 @type timeout: float 886 @return: the JSON response (dict format) 887 @rtype: dict 888 @raise WebService.UnsupportedParameters: if input parameters are invalid 889 @raise WebService.RequestError: if request cannot be satisfied 890 @raise WebService.MethodNotAvailable: if API method is not available 891 remotely and an error occurred (error code passed as exception 892 argument) 893 @raise WebService.AuthenticationRequired: if require_credentials is True 894 and credentials are required. 895 @raise WebService.AuthenticationFailed: if credentials are not valid 896 @raise WebService.MalformedResponse: if JSON response cannot be 897 converted back to dict. 898 @raise WebService.UnsupportedAPILevel: if client API and Web Service 899 API do not match 900 @raise WebService.MethodResponseError; if method execution failed 901 @raise WebService.CacheMiss: if cached=True and cached object is not 902 available 903 """ 904 cache_key = self._get_cache_key(func_name, params) 905 if cache or cached: 906 # this does call: _setup_generic_params() 907 obj = self._method_cached(func_name, params, cache_key = cache_key) 908 if (obj is None) and cached: 909 if const_debug_enabled(): 910 const_debug_write(__name__, 911 "WebService.%s(%s) = cache miss: %s" % ( 912 func_name, params, cache_key,)) 913 raise WebService.CacheMiss( 914 WebService.WEB_SERVICE_NOT_FOUND_CODE, method = func_name) 915 if obj is not None: 916 if const_debug_enabled(): 917 const_debug_write(__name__, 918 "WebService.%s(%s) = CACHED!" % ( 919 func_name, params,)) 920 return obj 921 if const_debug_enabled(): 922 const_debug_write(__name__, "WebService.%s(%s) = NOT cached" % ( 923 func_name, params,)) 924 else: 925 self._setup_generic_params(params) 926 927 if require_credentials: 928 # this can raise AuthenticationRequired 929 self._setup_credentials(params) 930 931 obj = None 932 try: 933 json_response, response = self._generic_post_handler(func_name, 934 params, file_params, timeout) 935 936 http_status = response.status 937 if http_status not in (httplib.OK,): 938 raise WebService.MethodNotAvailable(http_status, 939 method = func_name) 940 941 # try to convert the JSON response 942 try: 943 data = json.loads(json_response) 944 except (ValueError, TypeError) as err: 945 raise WebService.MalformedResponse(err, 946 method = func_name) 947 948 # check API 949 if data.get("api_rev") != WebService.SUPPORTED_API_LEVEL: 950 raise WebService.UnsupportedAPILevel(data['api_rev'], 951 method = func_name, message = data.get("message")) 952 953 code = data.get("code", -1) 954 if code == WebService.WEB_SERVICE_INVALID_CREDENTIALS_CODE: 955 # invalid credentials, ask again login data 956 raise WebService.AuthenticationFailed(code, 957 method = func_name, message = data.get("message")) 958 if code != WebService.WEB_SERVICE_RESPONSE_CODE_OK: 959 raise WebService.MethodResponseError(code, 960 method = func_name, message = data.get("message")) 961 962 if "r" not in data: 963 raise WebService.MalformedResponse("r not found", 964 method = func_name, message = data.get("message")) 965 obj = data["r"] 966 967 if const_debug_enabled(): 968 const_debug_write(__name__, "WebService.%s(%s) = fetched!" % ( 969 func_name, params,)) 970 return obj 971 972 finally: 973 if obj is not None: 974 # store cache 975 self._set_cached(cache_key, obj)
976
977 - def service_available(self, cache = True, cached = False):
978 """ 979 Return whether the Web Service is correctly able to answer our requests. 980 981 @keyword cache: True means use on-disk cache if available? 982 @type cache: bool 983 @keyword cached: if True, it will only use the on-disk cached call 984 result and raise WebService.CacheMiss if not found. 985 @type cached: bool 986 @return: True, if service is available 987 @rtype: bool 988 989 @raise WebService.UnsupportedParameters: if input parameters are invalid 990 @raise WebService.RequestError: if request cannot be satisfied 991 @raise WebService.MethodNotAvailable: if API method is not available 992 remotely and an error occurred (error code passed as exception 993 argument) 994 @raise WebService.AuthenticationRequired: if require_credentials is True 995 and credentials are required. 996 @raise WebService.AuthenticationFailed: if credentials are not valid 997 @raise WebService.MalformedResponse: if JSON response cannot be 998 converted back to dict. 999 @raise WebService.UnsupportedAPILevel: if client API and Web Service 1000 API do not match 1001 @raise WebService.MethodResponseError; if method execution failed 1002 @raise WebService.CacheMiss: if cached=True and cached object is not 1003 available 1004 """ 1005 params = locals().copy() 1006 params.pop("self") 1007 params.pop("cache") 1008 return self._method_getter("service_available", params, cache = cache, 1009 cached = cached, require_credentials = False)
1010
1011 - def data_send_available(self):
1012 """ 1013 Return whether data send is correctly working. A temporary file with 1014 random content is sent to the service, that would need to calculate 1015 its md5 hash. For security reason, data will be accepted remotely if, 1016 and only if its size is < 256 bytes. 1017 """ 1018 md5 = hashlib.md5() 1019 test_str = const_convert_to_rawstring("") 1020 for x in range(256): 1021 test_str += chr(x) 1022 md5.update(test_str) 1023 expected_hash = md5.hexdigest() 1024 func_name = "data_send_available" 1025 1026 tmp_fd, tmp_path = const_mkstemp(prefix="data_send_available") 1027 try: 1028 with os.fdopen(tmp_fd, "ab+") as tmp_f: 1029 tmp_f.write(test_str) 1030 tmp_f.seek(0) 1031 params = { 1032 "test_param": "hello", 1033 } 1034 file_params = { 1035 "test_file": ("test_file.txt", tmp_f), 1036 } 1037 remote_hash = self._method_getter(func_name, params, 1038 cache = False, require_credentials = False, 1039 file_params = file_params) 1040 finally: 1041 os.remove(tmp_path) 1042 1043 const_debug_write(__name__, 1044 "WebService.%s, expected: %s, got: %s" % ( 1045 func_name, repr(expected_hash), repr(remote_hash),)) 1046 return expected_hash == remote_hash
1047
1048 1049 -class AuthenticationStorage(Singleton):
1050 """ 1051 Entropy Web Service authentication credentials storage class. 1052 """ 1053 1054 _AUTH_FILE = ".entropy/id_entropy" 1055
1056 - def init_singleton(self):
1057 1058 self.__dump_lock = threading.Lock() 1059 # not loaded, load at very last moment 1060 self.__store = None
1061
1062 - def _get_authfile(self):
1063 """ 1064 Try to get the auth file. If it fails, return None. 1065 """ 1066 # setup auth file path 1067 home = os.getenv("HOME") 1068 auth_file = None 1069 if home is not None: 1070 if const_dir_writable(home): 1071 auth_file = os.path.join(home, 1072 AuthenticationStorage._AUTH_FILE) 1073 auth_dir = os.path.dirname(auth_file) 1074 if not os.path.isdir(auth_dir): 1075 try: 1076 os.makedirs(auth_dir, 0o700) 1077 const_setup_file(auth_dir, etpConst['entropygid'], 1078 0o700) 1079 except (OSError, IOError): 1080 # ouch, no permissions 1081 auth_file = None 1082 return auth_file
1083 1084 @property
1085 - def _authstore(self):
1086 """ 1087 Authentication data object automatically loaded from disk if needed. 1088 """ 1089 if self.__store is None: 1090 auth_file = self._get_authfile() 1091 store = {} 1092 if auth_file is not None: 1093 store = entropy.dump.loadobj(auth_file, complete_path = True) 1094 if store is None: 1095 store = {} 1096 elif not isinstance(store, dict): 1097 store = {} 1098 self.__store = store 1099 return self.__store
1100
1101 - def save(self):
1102 """ 1103 Save currently loaded authentication configuration to disk. 1104 1105 @return: True, if save was effectively run 1106 @rtype: bool 1107 """ 1108 with self.__dump_lock: 1109 auth_file = self._get_authfile() 1110 if auth_file is not None: 1111 entropy.dump.dumpobj(auth_file, self._authstore, 1112 complete_path = True, custom_permissions = 0o600) 1113 # make sure 1114 if auth_file is not None: 1115 try: 1116 const_setup_file(auth_file, etpConst['entropygid'], 1117 0o600) 1118 return True 1119 except (OSError, IOError): 1120 return False
1121
1122 - def add(self, repository_id, username, password):
1123 """ 1124 Add authentication credentials to Authentication configuration. 1125 1126 @param repository_id: repository identifier 1127 @type repository_id: string 1128 @param username: the username 1129 @type username: string 1130 @param password: the password 1131 @type password: string 1132 """ 1133 self._authstore[repository_id] = { 1134 'username': username, 1135 'password': password, 1136 }
1137
1138 - def remove(self, repository_id):
1139 """ 1140 Remove any credential for given repository identifier. 1141 1142 @param repository_id: repository identifier 1143 @type repository_id: string 1144 @return: True, if removal went fine (if there was something to remove) 1145 @rtype: bool 1146 """ 1147 try: 1148 del self._authstore[repository_id] 1149 return True 1150 except KeyError: 1151 return False
1152
1153 - def get(self, repository_id):
1154 """ 1155 Get authentication credentials for given repository identifier. 1156 1157 @return: tuple composed by username, password, or None, if credentials 1158 are not found 1159 @rtype: tuple or None 1160 """ 1161 data = self._authstore.get(repository_id) 1162 if data is None: 1163 return None 1164 return data['username'], data['password']
1165