feat(cmdb-api): CI password data store (#242)

* add secrets,for test

* feat: vault SDK (#238)

* feat: vault SDK

* docs: i18n

* perf(vault): format code

* feat(secrets): support vault

* feat: add inner password storage

* feat: secrets

* feat: add inner password storage

* feat: add secrets feature

* perf(secrets): review

---------

Co-authored-by: fxiang21 <fxiang21@126.com>
Co-authored-by: Mimo <osatmnzn@gmail.com>

Author: 3 people

cmdb-api/Pipfile

@@ -26,17 +26,17 @@ Flask-Bcrypt = "==1.0.1"
 Flask-Cors = ">=3.0.8"
 ldap3 = "==2.9.1"
 pycryptodome = "==3.12.0"
-cryptography = "==41.0.2"
+cryptography = ">=41.0.2"
 # Caching
 Flask-Caching = ">=1.0.0"
 # Environment variable parsing
 environs = "==4.2.0"
 marshmallow = "==2.20.2"
 # async tasks
-celery = "==5.3.1"
+celery = ">=5.3.1"
 celery_once = "==3.0.1"
 more-itertools = "==5.0.0"
-kombu = "==5.3.1"
+kombu = ">=5.3.1"
 # common setting
 timeout-decorator = "==0.5.0"
 WTForms = "==3.0.0"
@@ -59,6 +59,9 @@ Jinja2 = "==3.1.2"
 jinja2schema = "==0.1.4"
 msgpack-python = "==0.5.6"
 alembic = "==1.7.7"
+hvac = "==2.0.0"
+colorama = ">=0.4.6"
+pycryptodomex = ">=3.19.0"
 
 [dev-packages]
 # Testing
@@ -75,4 +78,3 @@ flake8-isort = "==2.7.0"
 isort = "==4.3.21"
 pep8-naming = "==0.8.2"
 pydocstyle = "==3.0.0"
-

cmdb-api/api/app.py

@@ -18,7 +18,9 @@
 
 import api.views.entry
 from api.extensions import (bcrypt, cache, celery, cors, db, es, login_manager, migrate, rd)
+from api.extensions import inner_secrets
 from api.flask_cas import CAS
+from api.lib.secrets.secrets import InnerKVManger
 from api.models.acl import User
 
 HERE = os.path.abspath(os.path.dirname(__file__))
@@ -126,6 +128,10 @@ def register_extensions(app):
     app.config.update(app.config.get("CELERY"))
     celery.conf.update(app.config)
 
+    if app.config.get('SECRETS_ENGINE') == 'inner':
+        with app.app_context():
+            inner_secrets.init_app(app, InnerKVManger())
+
 
 def register_blueprints(app):
     for item in getmembers(api.views.entry):

cmdb-api/api/commands/click_cmdb.py

@@ -7,6 +7,7 @@
 import time
 
 import click
+import requests
 from flask import current_app
 from flask.cli import with_appcontext
 from flask_login import login_user
@@ -29,6 +30,9 @@
 from api.lib.perm.acl.resource import ResourceTypeCRUD
 from api.lib.perm.acl.role import RoleCRUD
 from api.lib.perm.acl.user import UserCRUD
+from api.lib.secrets.inner import KeyManage
+from api.lib.secrets.inner import global_key_threshold
+from api.lib.secrets.secrets import InnerKVManger
 from api.models.acl import App
 from api.models.acl import ResourceType
 from api.models.cmdb import Attribute
@@ -53,6 +57,7 @@ def cmdb_init_cache():
     if relations:
         rd.create_or_update(relations, REDIS_PREFIX_CI_RELATION)
 
+    es = None
     if current_app.config.get("USE_ES"):
         from api.extensions import es
         from api.models.cmdb import Attribute
@@ -311,3 +316,128 @@ def cmdb_index_table_upgrade():
         CIIndexValueDateTime.create(ci_id=i.ci_id, attr_id=i.attr_id, value=i.value, commit=False)
         i.delete(commit=False)
     db.session.commit()
+
+
+@click.command()
+@click.option(
+    '-a',
+    '--address',
+    help='inner cmdb api, http://127.0.0.1:8000',
+)
+@with_appcontext
+def cmdb_inner_secrets_init(address):
+    """
+    init inner secrets for password feature
+    """
+    KeyManage(backend=InnerKVManger).init()
+
+    if address and address.startswith("http") and current_app.config.get("INNER_TRIGGER_TOKEN", "") != "":
+        resp = requests.post("{}/api/v0.1/secrets/auto_seal".format(address.strip("/")),
+                             headers={"Inner-Token": current_app.config.get("INNER_TRIGGER_TOKEN", "")})
+        if resp.status_code == 200:
+            KeyManage.print_response(resp.json())
+        else:
+            KeyManage.print_response({"message": resp.text, "status": "failed"})
+
+
+@click.command()
+@click.option(
+    '-a',
+    '--address',
+    help='inner cmdb api, http://127.0.0.1:8000',
+    required=True,
+)
+@with_appcontext
+def cmdb_inner_secrets_unseal(address):
+    """
+    unseal the secrets feature
+    """
+    address = "{}/api/v0.1/secrets/unseal".format(address.strip("/"))
+    if not address.startswith("http"):
+        KeyManage.print_response({"message": "invalid address, should start with http", "status": "failed"})
+        return
+    for i in range(global_key_threshold):
+        token = click.prompt(f'Enter unseal token {i + 1}', hide_input=True, confirmation_prompt=False)
+        assert token is not None
+        resp = requests.post(address, headers={"Unseal-Token": token})
+        if resp.status_code == 200:
+            KeyManage.print_response(resp.json())
+        else:
+            KeyManage.print_response({"message": resp.text, "status": "failed"})
+            return
+
+
+@click.command()
+@click.option(
+    '-a',
+    '--address',
+    help='inner cmdb api, http://127.0.0.1:8000',
+    required=True,
+)
+@click.option(
+    '-k',
+    '--token',
+    help='root token',
+    prompt=True,
+    hide_input=True,
+)
+@with_appcontext
+def cmdb_inner_secrets_seal(address, token):
+    """
+    seal the secrets feature
+    """
+    assert address is not None
+    assert token is not None
+    if address.startswith("http"):
+        address = "{}/api/v0.1/secrets/seal".format(address.strip("/"))
+        resp = requests.post(address, headers={
+            "Inner-Token": token,
+        })
+        if resp.status_code == 200:
+            KeyManage.print_response(resp.json())
+        else:
+            KeyManage.print_response({"message": resp.text, "status": "failed"})
+
+
+@click.command()
+@with_appcontext
+def cmdb_password_data_migrate():
+    """
+    Migrate CI password data, version >= v2.3.6
+    """
+    from api.models.cmdb import CIIndexValueText
+    from api.models.cmdb import CIValueText
+    from api.lib.secrets.inner import InnerCrypt
+    from api.lib.secrets.vault import VaultClient
+
+    attrs = Attribute.get_by(to_dict=False)
+    for attr in attrs:
+        if attr.is_password:
+
+            value_table = CIIndexValueText if attr.is_index else CIValueText
+
+            for i in value_table.get_by(attr_id=attr.id, to_dict=False):
+                if current_app.config.get("SECRETS_ENGINE", 'inner') == 'inner':
+                    _, status = InnerCrypt().decrypt(i.value)
+                    if status:
+                        continue
+
+                    encrypt_value, status = InnerCrypt().encrypt(i.value)
+                    if status:
+                        CIValueText.create(ci_id=i.ci_id, attr_id=attr.id, value=encrypt_value)
+                    else:
+                        continue
+                elif current_app.config.get("SECRETS_ENGINE") == 'vault':
+                    if i.value == '******':
+                        continue
+
+                    vault = VaultClient(current_app.config.get('VAULT_URL'), current_app.config.get('VAULT_TOKEN'))
+                    try:
+                        vault.update("/{}/{}".format(i.ci_id, i.attr_id), dict(v=i.value))
+                    except Exception as e:
+                        print('save password to vault failed: {}'.format(e))
+                        continue
+                else:
+                    continue
+
+                i.delete()

cmdb-api/api/extensions.py

@@ -12,6 +12,9 @@
 from api.lib.utils import ESHandler
 from api.lib.utils import RedisHandler
 
+from api.lib.secrets.inner import KeyManage
+
+
 bcrypt = Bcrypt()
 login_manager = LoginManager()
 db = SQLAlchemy(session_options={"autoflush": False})
@@ -21,3 +24,4 @@
 cors = CORS(supports_credentials=True)
 rd = RedisHandler()
 es = ESHandler()
+inner_secrets = KeyManage()