Merge pull request #2189 from openvstorage/develop

Promote

Author: JeffreyDevloo

ovs/constants/s3.py

@@ -0,0 +1,17 @@
+# Copyright (C) 2018 iNuron NV
+#
+# This file is part of Open vStorage Open Source Edition (OSE),
+# as available from
+#
+#      http://www.openvstorage.org and
+#      http://www.openvstorage.com.
+#
+# This file is free software; you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License v3 (GNU AGPLv3)
+# as published by the Free Software Foundation, in version 3 as it comes
+# in the LICENSE.txt file of the Open vStorage OSE distribution.
+#
+# Open vStorage is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY of any kind.
+
+S3_BASE = '/ovs/framework/hosts/{0}/s3'

ovs/dal/hybrids/servicetype.py

@@ -29,7 +29,8 @@ class ServiceType(DataObject):
                                                            'ARAKOON': 'Arakoon',
                                                            'ALBA_MGR': 'AlbaManager',
                                                            'MD_SERVER': 'MetadataServer',
-                                                           'ALBA_PROXY': 'AlbaProxy'})
+                                                           'ALBA_PROXY': 'AlbaProxy',
+                                                           'ALBA_S3_TRANSACTION': 'AlbaS3Transaction'})
     ARAKOON_CLUSTER_TYPES = DataObject.enumerator('Arakoon_cluster_type', ['ABM', 'FWK', 'NSM', 'SD', 'CFG'])
 
     __properties = [Property('name', str, unique=True, indexed=True, doc='Name of the ServiceType.')]

ovs/extensions/generic/configuration.py

@@ -24,17 +24,16 @@
 import string
 # noinspection PyUnresolvedReferences
 from ovs_extensions.caching.decorators import cache_file
+# ConnectionException, NotFoundException are here for backwards compatibility
 from ovs_extensions.generic.configuration import Configuration as _Configuration, ConnectionException, NotFoundException
+from ovs_extensions.constants.config import CONFIG_STORE_LOCATION
+from ovs.extensions.generic.system import System
 
 
 class Configuration(_Configuration):
     """
     Extends the 'default' configuration class
     """
-    ARAKOON_NAME = 'cacc'
-    ARAKOON_NAME_UNITTEST = 'unittest-cacc'
-    CACC_LOCATION = '/opt/OpenvStorage/config/arakoon_cacc.ini'
-    CONFIG_STORE_LOCATION = '/opt/OpenvStorage/config/framework.json'
 
     base_config = {'cluster_id': None,
                    'external_config': None,
@@ -125,6 +124,7 @@ def initialize(cls, external_config=None, logging_target=None):
                     base_cfg['messagequeue'][key] = value
         for key, value in base_cfg.iteritems():
             cls.set('/ovs/framework/{0}'.format(key), value, raw=False)
+        cls.register_usage(System.get_component_identifier())
 
     @classmethod
     @cache_file(CONFIG_STORE_LOCATION)
@@ -135,7 +135,7 @@ def read_store_info(cls):
         :return: The configured store
         :rtype: str
         """
-        with open(cls.CONFIG_STORE_LOCATION) as config_file:
+        with open(CONFIG_STORE_LOCATION) as config_file:
             contents = json.load(config_file)
             return contents['configuration_store']
 

ovs/extensions/generic/graphiteclient.py

@@ -14,8 +14,6 @@
 # Open vStorage is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY of any kind.
 
-import time
-import socket
 from ovs.extensions.generic.configuration import Configuration
 from ovs_extensions.generic.configuration.exceptions import ConfigurationNotFoundException as NotFoundException
 from ovs_extensions.generic.graphiteclient import GraphiteClient as _graphite_client
@@ -39,14 +37,14 @@ def __init__(self, ip=None, port=None, database=None):
         :param database: name of the database
         :type database: str
         """
+        graphite_data = {}
         if all(p is None for p in [ip, port]):
             # Nothing specified
-            try:
-                graphite_data = Configuration.get(self.CONFIG_PATH)
-            except NotFoundException:
+            graphite_data = self.get_graphite_config()
+            if not graphite_data:
                 raise RuntimeError('No graphite data found in config path `{0}`'.format(self.CONFIG_PATH))
 
-        ip = ip or graphite_data['host']
+        ip = ip or graphite_data['ip']
         port = port or graphite_data.get('port', 2003)
 
         ExtensionsToolbox.verify_required_params(verify_keys=True,
@@ -86,3 +84,17 @@ def send_statsmonkey_data(self, sm_data, function_name):
             for fieldkey, fieldvalue in datapointset['fields'].items():
                 tmp_path = '{0}.{1}'.format(path, fieldkey)
                 self.send(tmp_path, fieldvalue)
+
+    @classmethod
+    def get_graphite_config(cls):
+        # type: () -> Dict[str, Union[str, int]]
+        """
+        Retrieve the graphite config (if any)
+        :return:
+        """
+        try:
+            graphite_data = Configuration.get(cls.CONFIG_PATH)
+            return {'ip': graphite_data['host'],
+                    'port': graphite_data.get('port', 2003)}
+        except NotFoundException:
+            return {}

ovs/extensions/generic/system.py

@@ -21,7 +21,6 @@
 import os
 from ovs_extensions.caching.decorators import cache_file
 from ovs_extensions.generic.system import System as _System
-from ovs.extensions.packages.packagefactory import PackageFactory
 
 
 class System(_System):
@@ -30,7 +29,6 @@ class System(_System):
     """
 
     OVS_ID_FILE = '/etc/openvstorage_id'
-    RELEASE_NAME_FILE = '/opt/OpenvStorage/config/release_name'
     _machine_id = {}
 
     def __init__(self):
@@ -60,24 +58,6 @@ def read_my_machine_id(cls):
         with open(cls.OVS_ID_FILE, 'r') as the_file:
             return the_file.read().strip()
 
-    @classmethod
-    def get_release_name(cls, client=None):
-        """
-        Retrieve the release name
-        :param client: Client on which to retrieve the release name
-        :type client: ovs_extensions.generic.sshclient.SSHClient
-        :return: The name of the release
-        :rtype: str
-        """
-        try:
-            if client is not None:
-                return client.run(['cat', cls.RELEASE_NAME_FILE]).strip()
-            with open(cls.RELEASE_NAME_FILE, 'r') as the_file:
-                return the_file.read().strip()
-        except:
-            manager = PackageFactory.get_manager()
-            return manager.get_release_name()
-
     @classmethod
     def get_my_storagerouter(cls):
         """
@@ -90,3 +70,13 @@ def get_my_storagerouter(cls):
         if storagerouter is None:
             raise RuntimeError('Could not find the local StorageRouter')
         return storagerouter
+
+    @staticmethod
+    def get_component_identifier():
+        # type: () -> str
+        """
+        Retrieve the identifier of the component
+        :return: The ID of the component
+        :rtype: str
+        """
+        return 'framework'

ovs/extensions/generic/watcher.py

@@ -25,6 +25,7 @@
 import pika
 import uuid
 import argparse
+from ovs_extensions.constants.config import ARAKOON_NAME, CACC_LOCATION
 from ovs_extensions.db.arakoon.pyrakoon.pyrakoon.compat import NoGuarantee
 from ovs.extensions.db.arakooninstaller import ArakoonInstaller, ArakoonClusterConfig
 from ovs.extensions.generic.configuration import Configuration
@@ -141,9 +142,9 @@ def services_running(self):
                     self.log_message('  Error during configuration store test: {0}'.format(ex), 2)
                     return False
 
-                with open(Configuration.CACC_LOCATION) as config_file:
+                with open(CACC_LOCATION) as config_file:
                     contents = config_file.read()
-                config = ArakoonClusterConfig(cluster_id=Configuration.ARAKOON_NAME, load_config=False)
+                config = ArakoonClusterConfig(cluster_id=ARAKOON_NAME, load_config=False)
                 config.read_config(contents=contents)
                 client = ArakoonInstaller.build_client(config)
                 contents = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY, consistency=NoGuarantee())
@@ -153,12 +154,12 @@ def services_running(self):
                     except Exception as ex:
                         self.log_message('  Configuration stored in configuration store seems to be corrupt: {0}'.format(ex), 2)
                         return False
-                    temp_filename = '{0}~'.format(Configuration.CACC_LOCATION)
+                    temp_filename = '{0}~'.format(CACC_LOCATION)
                     with open(temp_filename, 'w') as config_file:
                         config_file.write(contents)
                         config_file.flush()
                         os.fsync(config_file)
-                    os.rename(temp_filename, Configuration.CACC_LOCATION)
+                    os.rename(temp_filename, CACC_LOCATION)
                     Watcher.LOG_CONTENTS = contents
                 self.log_message('  Configuration store OK', 0)
 

ovs/extensions/packages/packagefactory.py

@@ -27,6 +27,8 @@ class PackageFactory(_PackageFactory):
     """
     Factory class returning specialized classes
     """
+    # Allow the user to add a custom release name
+    RELEASE_NAME_FILE = '/opt/OpenvStorage/config/release_name'
     _logger = Logger('extensions-packages')
 
     def __init__(self):
@@ -91,3 +93,21 @@ def get_package_info(cls):
                     'mutually_exclusive': {cls.PKG_VOLDRV_BASE, cls.PKG_VOLDRV_SERVER}}
         else:
             raise ValueError('Unsupported edition found: "{0}"'.format(edition))
+
+    @classmethod
+    def get_release_name(cls, client=None):
+        """
+        Retrieve the release name
+        :param client: Client on which to retrieve the release name
+        :type client: ovs_extensions.generic.sshclient.SSHClient
+        :return: The name of the release
+        :rtype: str
+        """
+        try:
+            if client is not None:
+                return client.run(['cat', cls.RELEASE_NAME_FILE]).strip()
+            with open(cls.RELEASE_NAME_FILE, 'r') as the_file:
+                return the_file.read().strip()
+        except:
+            manager = cls.get_manager()
+            return manager.get_release_name()

ovs/extensions/plugins/apiclient.py

@@ -74,7 +74,25 @@ def __init__(self, ip, port, credentials, timeout, log_min_duration=1):
         self.timeout = timeout
         self._log_min_duration = log_min_duration
 
-    def _call(self, method, url, data=None, timeout=None, clean=False):
+    def _call(self, method, url, data=None, json=None, timeout=None, clean=False):
+        # type: (callable, str, dict, dict, int, bool) -> any
+        """
+        Calls the provided function and adds headings
+        :param method: Method to call
+        :type method: callable
+        :param url: Url to call
+        :type url: str
+        :param data: Data to provide. This parameter will not set the JSON header so data may be interpreted differently!
+        :type data: dict
+        :param json: Data to provide as JSON parameters. This parameter will set the JSOn header to data will be interpreted as a JSON string
+        :type json: dict
+        :param timeout: Timeout to wait for a reply of the server
+        :type timeout: int
+        :param clean: Should the data be cleaned (metdata entries stripped from the result)
+        :type clean: bool
+        :return: The response
+        :rtype: any
+        """
         if timeout is None:
             timeout = self.timeout
 
@@ -86,8 +104,13 @@ def _call(self, method, url, data=None, timeout=None, clean=False):
                   'headers': self._base_headers,
                   'verify': False,
                   'timeout': timeout}
-        if data is not None:
-            kwargs['data'] = data
+        # Requests library can both take in 'data' or 'json' keyword.
+        # When data is given: no extra heading is added and the data is serialized as json
+        # When json is given: the 'Content type: Application/json' header is added.
+        # The loop is to provide both options
+        for key, val in [('json', json), ('data', data)]:
+            if val is not None:
+                kwargs[key] = val
         response = method(**kwargs)
         if response.status_code == 404:
             msg = 'URL not found: {0}'.format(kwargs['url'])
@@ -121,7 +144,7 @@ def _refresh(self):
         return self._base_url, self._base_headers
 
     def extract_data(self, response_data, old_key=None):
-        # type: (dict) -> any
+        # type: (dict, Optional[str]) -> any
         """
         Extract the data from the API
         For backwards compatibility purposes (older asd-managers might not wrap their data)
@@ -158,3 +181,83 @@ def clean(cls, data):
             elif isinstance(data_copy[key], dict):
                 data_copy[key] = cls.clean(data_copy[key])
         return data_copy
+
+    def get(self, url, data=None, json=None, timeout=None, clean=False):
+        """
+        Executes a GET call
+        :param url: Url to call
+        :type url: str
+        :param data: Data to provide. This parameter will not set the JSON header so data may be interpreted differently!
+        :type data: dict
+        :param json: Data to provide as JSON parameters. This parameter will set the JSOn header to data will be interpreted as a JSON string
+        :type json: dict
+        :param timeout: Timeout to wait for a reply of the server
+        :type timeout: int
+        :param clean: Should the data be cleaned (metadata entries stripped from the result)
+        :type clean: bool
+        """
+        return self._call(method=requests.get, url=url, data=data, json=json, clean=clean, timeout=timeout)
+
+    def post(self, url, data=None, json=None, timeout=None, clean=False):
+        """
+        Executes a POST call
+        :param url: Url to call
+        :type url: str
+        :param data: Data to provide. This parameter will not set the JSON header so data may be interpreted differently!
+        :type data: dict
+        :param json: Data to provide as JSON parameters. This parameter will set the JSOn header to data will be interpreted as a JSON string
+        :type json: dict
+        :param timeout: Timeout to wait for a reply of the server
+        :type timeout: int
+        :param clean: Should the data be cleaned (metadata entries stripped from the result)
+        :type clean: bool
+        """
+        return self._call(method=requests.post, url=url, data=data, json=json, clean=clean, timeout=timeout)
+
+    def put(self, url, data=None, json=None, timeout=None, clean=False):
+        """
+        Executes a PUT call
+        :param url: Url to call
+        :type url: str
+        :param data: Data to provide. This parameter will not set the JSON header so data may be interpreted differently!
+        :type data: dict
+        :param json: Data to provide as JSON parameters. This parameter will set the JSOn header to data will be interpreted as a JSON string
+        :type json: dict
+        :param timeout: Timeout to wait for a reply of the server
+        :type timeout: int
+        :param clean: Should the data be cleaned (metadata entries stripped from the result)
+        :type clean: bool
+        """
+        return self._call(method=requests.put, url=url, data=data, json=json, clean=clean, timeout=timeout)
+
+    def patch(self, url, data=None, json=None, timeout=None, clean=False):
+        """
+        Executes a PATCH call
+        :param url: Url to call
+        :type url: str
+        :param data: Data to provide. This parameter will not set the JSON header so data may be interpreted differently!
+        :type data: dict
+        :param json: Data to provide as JSON parameters. This parameter will set the JSOn header to data will be interpreted as a JSON string
+        :type json: dict
+        :param timeout: Timeout to wait for a reply of the server
+        :type timeout: int
+        :param clean: Should the data be cleaned (metadata entries stripped from the result)
+        :type clean: bool
+        """
+        return self._call(method=requests.patch, url=url, data=data, json=json, clean=clean, timeout=timeout)
+
+    def delete(self, url, data=None, json=None, timeout=None, clean=False):
+        """
+        Executes a DELETE call
+        :param url: Url to call
+        :type url: str
+        :param data: Data to provide. This parameter will not set the JSON header so data may be interpreted differently!
+        :type data: dict
+        :param json: Data to provide as JSON parameters. This parameter will set the JSOn header to data will be interpreted as a JSON string
+        :type json: dict
+        :param timeout: Timeout to wait for a reply of the server
+        :type timeout: int
+        :param clean: Should the data be cleaned (metadata entries stripped from the result)
+        :type clean: bool
+        """
+        return self._call(method=requests.delete, url=url, data=data, json=json, clean=clean, timeout=timeout)

ovs/extensions/support/agent.py

@@ -34,9 +34,9 @@
 from ovs.extensions.generic.system import System
 from ovs.extensions.packages.packagefactory import PackageFactory
 from ovs.extensions.services.servicefactory import ServiceFactory
+from ovs_extensions.constants.config import CONFIG_STORE_LOCATION
 from ovs.lib.helpers.toolbox import Toolbox
 
-
 class ConfigurationNotFoundError(RuntimeError):
     """
     Raised if the config key could not be found
@@ -120,7 +120,7 @@ def __init__(self):
             raise NotImplementedError('Only Systemd is supported')
 
         # Potential failing calls
-        self._cluster_id = self.get_config_key(self.LOCATION_CLUSTER_ID, fallback=[Configuration.CONFIG_STORE_LOCATION, 'cluster_id'])
+        self._cluster_id = self.get_config_key(self.LOCATION_CLUSTER_ID, fallback=[CONFIG_STORE_LOCATION, 'cluster_id'])
         self.interval = self.get_config_key(self.LOCATION_INTERVAL, fallback=[self.FALLBACK_CONFIG, self.KEY_INTERVAL], default=self.DEFAULT_INTERVAL)
         self._openvpn_service_name = 'openvpn@ovs_{0}-{1}'.format(self._cluster_id, self._node_id)