Merge branch 'master' into ek/fix-concretize-syscall-args

* master:
  Fix inheritance magic and add plugin enable/disable (#1696)
  Add hashing to constraintSets (#1703)
  Reduce log messages (#1700)
  Updated Proc error message to be more informative (#1704)
  Move states that selfdestruct to the ready queue. (#1699)
  Reduce concrete hash log messages. (#1695)

Author: ekilmer

manticore/core/manticore.py

@@ -79,17 +79,65 @@ def to_class(self):
 
 
 class ManticoreBase(Eventful):
-    def __new__(cls, *args, **kwargs):
-        if cls in (ManticoreBase, ManticoreSingle, ManticoreThreading, ManticoreMultiprocessing):
-            raise ManticoreError("Should not instantiate this")
+    def _manticore_single(self):
+        self._worker_type = WorkerSingle
 
-        cl = consts.mprocessing.to_class()
-        # change ManticoreBase for the more specific class
-        bases = {cl if issubclass(base, ManticoreBase) else base for base in cls.__bases__}
-        cls.__bases__ = tuple(bases)
+        class FakeLock:
+            def _nothing(self, *args, **kwargs):
+                pass
 
-        random.seed(consts.seed)
-        return super().__new__(cls)
+            acquire = _nothing
+            release = _nothing
+            __enter__ = _nothing
+            __exit__ = _nothing
+            notify_all = _nothing
+            wait = _nothing
+
+            def wait_for(self, condition, *args, **kwargs):
+                if not condition():
+                    raise Exception("Deadlock: Waiting for CTRL+C")
+
+        self._lock = FakeLock()
+        self._killed = ctypes.c_bool(False)
+        self._running = ctypes.c_bool(False)
+        self._ready_states = []
+        self._terminated_states = []
+        self._busy_states = []
+        self._killed_states = []
+        self._shared_context = {}
+
+    def _manticore_threading(self):
+        self._worker_type = WorkerThread
+        self._lock = threading.Condition()
+        self._killed = ctypes.c_bool(False)
+        self._running = ctypes.c_bool(False)
+        self._ready_states = []
+        self._terminated_states = []
+        self._busy_states = []
+        self._killed_states = []
+        self._shared_context = {}
+
+    def _manticore_multiprocessing(self):
+        def raise_signal():
+            signal.signal(signal.SIGINT, signal.SIG_IGN)
+
+        self._worker_type = WorkerProcess
+        # This is the global manager that will handle all shared memory access
+        # See. https://docs.python.org/3/library/multiprocessing.html#multiprocessing.managers.SyncManager
+        self._manager = SyncManager()
+        self._manager.start(raise_signal)
+        # The main manticore lock. Acquire this for accessing shared objects
+        # THINKME: we use the same lock to access states lists and shared contexts
+        self._lock = self._manager.Condition()
+        self._killed = self._manager.Value(bool, False)
+        self._running = self._manager.Value(bool, False)
+        # List of state ids of States on storage
+        self._ready_states = self._manager.list()
+        self._terminated_states = self._manager.list()
+        self._busy_states = self._manager.list()
+        self._killed_states = self._manager.list()
+        self._shared_context = self._manager.dict()
+        self._context_value_types = {list: self._manager.list, dict: self._manager.dict}
 
     # Decorators added first for convenience.
     def sync(func: Callable) -> Callable:  # type: ignore
@@ -255,6 +303,11 @@ def __init__(self, initial_state, workspace_url=None, outputspace_url=None, **kw
         :param kwargs: other kwargs, e.g.
         """
         super().__init__()
+        random.seed(consts.seed)
+        {consts.mprocessing.single: self._manticore_single,
+         consts.mprocessing.threading: self._manticore_threading,
+         consts.mprocessing.multiprocessing: self._manticore_multiprocessing
+         }[consts.mprocessing]()
 
         if any(
             not hasattr(self, x)
@@ -1008,82 +1061,3 @@ def save_run_data(self):
             config.save(f)
 
         logger.info("Results in %s", self._output.store.uri)
-
-
-class ManticoreSingle(ManticoreBase):
-    _worker_type = WorkerSingle
-
-    def __init__(self, *args, **kwargs):
-        class FakeLock:
-            def _nothing(self, *args, **kwargs):
-                pass
-
-            acquire = _nothing
-            release = _nothing
-            __enter__ = _nothing
-            __exit__ = _nothing
-            notify_all = _nothing
-            wait = _nothing
-
-            def wait_for(self, condition, *args, **kwargs):
-                if not condition():
-                    raise Exception("Deadlock: Waiting for CTRL+C")
-
-        self._lock = FakeLock()
-        self._killed = ctypes.c_bool(False)
-        self._running = ctypes.c_bool(False)
-
-        self._ready_states = []
-        self._terminated_states = []
-        self._busy_states = []
-        self._killed_states = []
-
-        self._shared_context = {}
-        super().__init__(*args, **kwargs)
-
-
-class ManticoreThreading(ManticoreBase):
-    _worker_type = WorkerThread
-
-    def __init__(self, *args, **kwargs):
-        self._lock = threading.Condition()
-        self._killed = ctypes.c_bool(False)
-        self._running = ctypes.c_bool(False)
-
-        self._ready_states = []
-        self._terminated_states = []
-        self._busy_states = []
-        self._killed_states = []
-
-        self._shared_context = {}
-
-        super().__init__(*args, **kwargs)
-
-
-def raise_signal():
-    signal.signal(signal.SIGINT, signal.SIG_IGN)
-
-
-class ManticoreMultiprocessing(ManticoreBase):
-    _worker_type = WorkerProcess
-
-    def __init__(self, *args, **kwargs):
-        # This is the global manager that will handle all shared memory access
-        # See. https://docs.python.org/3/library/multiprocessing.html#multiprocessing.managers.SyncManager
-        self._manager = SyncManager()
-        self._manager.start(raise_signal)
-        # The main manticore lock. Acquire this for accessing shared objects
-        # THINKME: we use the same lock to access states lists and shared contexts
-        self._lock = self._manager.Condition()
-        self._killed = self._manager.Value(bool, False)
-        self._running = self._manager.Value(bool, False)
-
-        # List of state ids of States on storage
-        self._ready_states = self._manager.list()
-        self._terminated_states = self._manager.list()
-        self._busy_states = self._manager.list()
-        self._killed_states = self._manager.list()
-        self._shared_context = self._manager.dict()
-        self._context_value_types = {list: self._manager.list, dict: self._manager.dict}
-
-        super().__init__(*args, **kwargs)

manticore/core/plugin.py

@@ -3,15 +3,51 @@
 import cProfile
 import pstats
 import threading
+from functools import wraps
 
 from .smtlib import issymbolic
 
 logger = logging.getLogger(__name__)
 
 
 class Plugin:
+
     def __init__(self):
         self.manticore = None
+        self._enabled_key = f"{str(type(self))}_enabled_{hash(self)}"
+        self._plugin_context_name = f"{str(type(self))}_context_{hash(self)}"
+        self.__decorate_callbacks()
+
+    def __decorate_callbacks(self):
+        for attr in self.__dict__:
+            if attr.endswith('_callback'):
+                method = getattr(self, attr)
+                if callable(method):
+                    setattr(self, attr, self._if_enabled(method))
+
+    def enable(self):
+        """ Enable all callbacks """
+        with self.manticore.locked_context() as context:
+            context[self._enabled_key] = True
+
+    def disable(self):
+        """ Disable all callbacks """
+        with self.manticore.locked_context() as context:
+            context[self._enabled_key] = False
+
+    def is_enabled(self):
+        """ True if callbacks are enabled """
+        with self.manticore.locked_context() as context:
+            return context.get(self._enabled_key, True)
+
+    @staticmethod
+    def _if_enabled(f):
+        """ decorator used to guard callbacks """
+        @wraps(f)
+        def g(self, *args, **kwargs):
+            if self.is_enabled():
+                return f(self, *args, **kwargs)
+        return g
 
     @property
     def name(self):
@@ -25,7 +61,7 @@ def locked_context(self, key=None, value_type=list):
         when parallel analysis is activated. Code within the `with` block is executed
         atomically, so access of shared variables should occur within.
         """
-        plugin_context_name = str(type(self))
+        plugin_context_name = self._plugin_context_name
         with self.manticore.locked_context(plugin_context_name, dict) as context:
             if key is None:
                 yield context
@@ -37,7 +73,7 @@ def locked_context(self, key=None, value_type=list):
     @property
     def context(self):
         """ Convenient access to shared context """
-        plugin_context_name = str(type(self))
+        plugin_context_name = self._plugin_context_name
         if plugin_context_name not in self.manticore.context:
             self.manticore.context[plugin_context_name] = {}
         return self.manticore.context[plugin_context_name]

manticore/core/smtlib/constraints.py

@@ -17,10 +17,16 @@
     Constant,
 )
 from .visitors import GetDeclarations, TranslatorSmtlib, get_variables, simplify, replace
+from ...utils import config
 import logging
 
 logger = logging.getLogger(__name__)
 
+consts = config.get_group("smt")
+consts.add(
+    "related_constraints", default=False, description="Try slicing the current path constraint to contain only related items"
+)
+
 
 class ConstraintException(SmtlibError):
     """
@@ -56,6 +62,9 @@ def __reduce__(self):
             },
         )
 
+    def __hash__(self):
+        return hash(self.constraints)
+
     def __enter__(self) -> "ConstraintSet":
         assert self._child is None
         self._child = self.__class__()
@@ -96,7 +105,6 @@ def add(self, constraint) -> None:
                 self._constraints = [constraint]
             else:
                 return
-
         self._constraints.append(constraint)
 
     def _get_sid(self) -> int:
@@ -118,7 +126,8 @@ def __get_related(self, related_to=None):
         # satisfiable one, {}.
         #   In light of the above, the core __get_related logic is currently disabled.
         # if related_to is not None:
-        if False:
+        # feliam: This assumes the previous constraints are already SAT (normal SE forking)
+        if consts.related_constraints and related_to is not None:
             number_of_constraints = len(self.constraints)
             remaining_constraints = set(self.constraints)
             related_variables = get_variables(related_to)

manticore/core/smtlib/solver.py

@@ -18,6 +18,7 @@
 import collections
 import shlex
 import time
+from functools import lru_cache
 from typing import Dict, Tuple
 from subprocess import PIPE, Popen
 import re
@@ -421,6 +422,7 @@ def _pop(self):
         """Recall the last pushed constraint store and state."""
         self._send("(pop 1)")
 
+    @lru_cache(maxsize=32)
     def can_be_true(self, constraints: ConstraintSet, expression: Union[bool, Bool] = True) -> bool:
         """Check if two potentially symbolic values can be equal"""
         if isinstance(expression, bool):
@@ -438,6 +440,7 @@ def can_be_true(self, constraints: ConstraintSet, expression: Union[bool, Bool]
             return self._is_sat()
 
     # get-all-values min max minmax
+    @lru_cache(maxsize=32)
     def get_all_values(self, constraints, expression, maxcnt=None, silent=False):
         """Returns a list with all the possible values for the symbol x"""
         if not isinstance(expression, Expression):

manticore/core/worker.py

@@ -132,7 +132,7 @@ def run(self, *args):
                         assert current_state is None
                     # Handling Forking and terminating exceptions
                     except Concretize as exc:
-                        logger.info("[%r] Performing %r", self.id, exc.message)
+                        logger.debug("[%r] Performing %r", self.id, exc.message)
                         # The fork() method can decides which state to keep
                         # exploring. For example when the fork results in a
                         # single state it is better to just keep going.

manticore/ethereum/manticore.py

@@ -871,10 +871,10 @@ def _transaction(self, sort, caller, value=0, address=None, data=None, gas=21000
             caller = int(caller)
         # Defaults, call data is empty
         if data is None:
-            data = bytearray(b"")
-        if isinstance(data, (str, bytes)):
-            data = bytearray(data)
-        if not isinstance(data, (bytearray, Array)):
+            data = b""
+        if isinstance(data, str):
+            data = bytes(data)
+        if not isinstance(data, (bytes, Array)):
             raise TypeError("code bad type")
 
         # Check types
@@ -1184,12 +1184,13 @@ def _on_unsound_symbolication(self, state, func, data, result):
         if value is not None:
             with self.locked_context("ethereum", dict) as ethereum_context:
                 global_known_pairs = ethereum_context.get(f"symbolic_func_conc_{name}", set())
-                global_known_pairs.add((data, value))
-                ethereum_context[f"symbolic_func_conc_{name}"] = global_known_pairs
+                if (data, value) not in global_known_pairs:
+                    global_known_pairs.add((data, value))
+                    ethereum_context[f"symbolic_func_conc_{name}"] = global_known_pairs
+                    logger.info(f"Found a concrete {name} {data} -> {value}")
             concrete_pairs = state.context.get(f"symbolic_func_conc_{name}", set())
             concrete_pairs.add((data, value))
             state.context[f"symbolic_func_conc_{name}"] = concrete_pairs
-            logger.info(f"Found a concrete {name} {data} -> {value}")
         else:
             # we can not calculate the concrete value lets use a fresh symbol
             with self.locked_context("ethereum", dict) as ethereum_context:
@@ -1425,16 +1426,13 @@ def _terminate_state_callback(self, state, e):
         # generate a testcase. FIXME This should be configurable as REVERT and
         # THROW; it actually changes the balance and nonce? of some accounts
 
-        if tx.result in {"SELFDESTRUCT", "REVERT", "THROW", "TXERROR"}:
+        if tx.return_value == 0:
             pass
-        elif tx.result in {"RETURN", "STOP"}:
+        else:
             # if not a revert, we save the state for further transactions
             with self.locked_context("ethereum.saved_states", list) as saved_states:
                 saved_states.append(state.id)
 
-        else:
-            logger.debug("Exception in state. Discarding it")
-
     # Callbacks
     def _did_evm_execute_instruction_callback(self, state, instruction, arguments, result):
         """ INTERNAL USE """

manticore/ethereum/plugins.py

@@ -1,7 +1,6 @@
 import sys
 
 from functools import reduce
-
 import re
 
 from ..core.plugin import Plugin

manticore/platforms/evm.py

@@ -2416,7 +2416,7 @@ def _open_transaction(self, sort, address, price, bytecode_or_data, caller, valu
         )
         if sort == "CREATE":
             bytecode = bytecode_or_data
-            data = bytearray()
+            data = bytes()
         else:
             bytecode = self.get_code(address)
             data = bytecode_or_data
@@ -2790,7 +2790,6 @@ def calculate_new_address(sender=None, nonce=None):
         return new_address
 
     def execute(self):
-
         self._process_pending_transaction()
         if self.current_vm is None:
             raise TerminateState("Trying to execute an empty transaction", testcase=False)