Provide a way to the validation key and data bag secret from S3

README.md

@@ -81,6 +81,36 @@ Additionally the following options may be set in your `knife.rb`:
 - distro
 - template_file
 
+Using Cloud-Based Secret Data
+-----------------------------
+knife-ec2 now includes the ability to retrieve the encrypted data bag secret and validation keys directly from a cloud-based assets store (currently on S3 is supported).  To enable this functionality, you must first upload keys to S3 and give them appropriate permissions.  The following is a suggested set of IAM permissions required to make this work:
+
+```json
+{
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "s3:Get*",
+        "s3:List*"
+      ],
+      "Resource": [
+        "arn:aws:s3:::provisioning.bucket.com/chef/*"
+      ]
+    }
+  ]
+}
+```
+
+### Use the following configuration options in `knife.rb` to set the source URLs:
+```ruby
+knife[:validation_key_url] = 's3://provisioning.bucket.com/chef/evertrue-validator.pem'
+knife[:s3_secret] = 's3://provisioning.bucket.com/chef/encrypted_data_bag_secret'
+```
+
+### Alternatively, URLs can be passed directly on the command line:
+- Validation Key: `--validation-key-url s3://provisioning.bucket.com/chef/evertrue-validator.pem`
+- Encrypted Data Bag Secret: `--s3-secret s3://provisioning.bucket.com/chef/encrypted_data_bag_secret`
 
 Subcommands
 -----------

lib/chef/knife/ec2_server_create.rb

@@ -18,6 +18,7 @@
 #
 
 require 'chef/knife/ec2_base'
+require 'chef/knife/s3_source'
 require 'chef/knife/winrm_base'
 
 class Chef
@@ -27,7 +28,9 @@ class Ec2ServerCreate < Knife
       include Knife::Ec2Base
       include Knife::WinrmBase
       deps do
+        require 'tempfile'
         require 'fog'
+        require 'uri'
         require 'readline'
         require 'chef/json_compat'
         require 'chef/knife/bootstrap'
@@ -194,6 +197,12 @@ class Ec2ServerCreate < Knife
         :description => "A file containing the secret key to use to encrypt data bag item values",
         :proc => lambda { |sf| Chef::Config[:knife][:secret_file] = sf }
 
+      option :s3_secret,
+        :long => '--s3-secret S3_SECRET_URL',
+        :description => 'S3 URL (e.g. s3://bucket/file) for the ' \
+          'encrypted_data_bag_secret_file',
+        :proc => lambda { |url| Chef::Config[:knife][:s3_secret] = url }
+
       option :json_attributes,
         :short => "-j JSON",
         :long => "--json-attributes JSON",
@@ -280,6 +289,11 @@ class Ec2ServerCreate < Knife
         :description => "The maximum time in minutes to wait to for authentication over the transport to the node to succeed. The default value is 25 minutes.",
         :default => 25
 
+      option :validation_key_url,
+        :long => "--validation-key-url URL",
+        :description => "Path to the validation key",
+        :proc => proc { |m| Chef::Config[:validation_key_url] = m }
+
       def run
         $stdout.sync = true
 
@@ -360,6 +374,11 @@ def run
         end
         msg_pair("Private IP Address", @server.private_ip_address)
 
+        if Chef::Config[:knife][:validation_key_url]
+          download_validation_key(validation_key_path)
+          Chef::Config[:validation_key] = validation_key_path
+        end
+
         #Check if Server is Windows or Linux
         if is_image_windows?
           protocol = locate_config_value(:bootstrap_protocol)
@@ -441,6 +460,44 @@ def run
         msg_pair("JSON Attributes",config[:json_attributes]) unless !config[:json_attributes] || config[:json_attributes].empty?
       end
 
+      def validation_key_path
+        @validation_key_path ||= begin
+          if URI(Chef::Config[:knife][:validation_key_url]).scheme == 'file'
+            URI(Chef::Config[:knife][:validation_key_url]).path
+          else
+            validation_key_tmpfile.path
+          end
+        end
+      end
+
+      def validation_key_tmpfile
+        @validation_key_tmpfile ||= Tempfile.new('validation_key')
+      end
+
+      def download_validation_key(tempfile)
+        Chef::Log.debug 'Downloading validation key ' \
+          "<#{Chef::Config[:knife][:validation_key_url]}> to file " \
+          "<#{tempfile}>"
+
+        case URI(Chef::Config[:knife][:validation_key_url]).scheme
+        when 's3'
+          File.open(tempfile, 'w') { |f| f.write(s3_validation_key) }
+        end
+      end
+
+      def s3_validation_key
+        @s3_validation_key ||= begin
+          Chef::Knife::S3Source.fetch(Chef::Config[:knife][:validation_key_url])
+        end
+      end
+
+      def s3_secret
+        @s3_secret ||= begin
+          return false unless locate_config_value(:s3_secret)
+          Chef::Knife::S3Source.fetch(locate_config_value(:s3_secret))
+        end
+      end
+
       def bootstrap_common_params(bootstrap)
         bootstrap.config[:run_list] = config[:run_list]
         bootstrap.config[:bootstrap_version] = locate_config_value(:bootstrap_version)
@@ -451,7 +508,7 @@ def bootstrap_common_params(bootstrap)
         bootstrap.config[:first_boot_attributes] = locate_config_value(:json_attributes) || {}
         bootstrap.config[:encrypted_data_bag_secret] = locate_config_value(:encrypted_data_bag_secret)
         bootstrap.config[:encrypted_data_bag_secret_file] = locate_config_value(:encrypted_data_bag_secret_file)
-        bootstrap.config[:secret] = locate_config_value(:secret)
+        bootstrap.config[:secret] = s3_secret || locate_config_value(:secret)
         bootstrap.config[:secret_file] = locate_config_value(:secret_file)
         # Modify global configuration state to ensure hint gets set by
         # knife-bootstrap

lib/chef/knife/s3_source.rb

@@ -0,0 +1,40 @@
+require 'fog'
+
+class Chef
+  class Knife
+    class S3Source
+      attr_accessor :url
+
+      def self.fetch(url)
+        source = Chef::Knife::S3Source.new
+        source.url = url
+        source.body
+      end
+
+      def body
+        bucket_obj.files.get(path).body
+      end
+
+      private
+
+      def bucket_obj
+        @bucket_obj ||= fog.directories.get(bucket)
+      end
+
+      def bucket
+        URI(@url).host
+      end
+
+      def path
+        URI(@url).path.sub(/^\//, '')
+      end
+
+      def fog
+        @fog ||= Fog::Storage::AWS.new(
+          aws_access_key_id: Chef::Config[:knife][:aws_access_key_id],
+          aws_secret_access_key: Chef::Config[:knife][:aws_secret_access_key]
+        )
+      end
+    end
+  end
+end

spec/unit/ec2_server_create_spec.rb

@@ -70,6 +70,15 @@
     @ec2_server_attribs.each_pair do |attrib, value|
       @new_ec2_server.stub(attrib).and_return(value)
     end
+
+    @s3_connection = double(Fog::Storage::AWS)
+
+    @bootstrap = Chef::Knife::Bootstrap.new
+    Chef::Knife::Bootstrap.stub(:new).and_return(@bootstrap)
+
+    @validation_key_url = 's3://bucket/foo/bar'
+    @validation_key_file = '/tmp/a_good_temp_file'
+    @validation_key_body = "TEST VALIDATION KEY\n"
   end
 
   describe "run" do
@@ -84,9 +93,6 @@
       @knife_ec2_create.stub(:puts)
       @knife_ec2_create.stub(:print)
       @knife_ec2_create.config[:image] = '12345'
-
-      @bootstrap = Chef::Knife::Bootstrap.new
-      Chef::Knife::Bootstrap.stub(:new).and_return(@bootstrap)
       @bootstrap.should_receive(:run)
     end
 
@@ -177,6 +183,19 @@
       @knife_ec2_create.ui.should_receive(:warn).with(/retrying/)
       @knife_ec2_create.run
     end
+
+    it 'actually writes to the validation key tempfile' do
+      @new_ec2_server.should_receive(:wait_for).and_return(true)
+      Chef::Config[:knife][:validation_key_url] =
+        @validation_key_url
+      @knife_ec2_create.config[:validation_key_url] =
+        @validation_key_url
+
+      @knife_ec2_create.stub_chain(:validation_key_tmpfile, :path).and_return(@validation_key_file)
+      Chef::Knife::S3Source.stub(:fetch).with(@validation_key_url).and_return(@validation_key_body)
+      File.should_receive(:open).with(@validation_key_file, 'w')
+      @knife_ec2_create.run
+    end
   end
 
   describe "run for EC2 Windows instance" do
@@ -340,6 +359,19 @@
         expect(bootstrap.config[:secret_file]).to eql("cli-provided-secret-file")
       end
     end
+
+    context 'S3-based secret' do
+      before(:each) do
+        Chef::Config[:knife][:s3_secret] =
+          's3://test.bucket/folder/encrypted_data_bag_secret'
+        @secret_content = "TEST DATA BAG SECRET\n"
+        @knife_ec2_create.stub(:s3_secret).and_return(@secret_content)
+      end
+
+      it 'sets the secret to the expected test string' do
+        expect(bootstrap.config[:secret]).to eql(@secret_content)
+      end
+    end
   end
 
   describe "when configuring the bootstrap process" do
@@ -535,6 +567,20 @@
       end
     end
 
+    it 'understands that file:// validation key URIs are just paths' do
+      Chef::Config[:knife][:validation_key_url] = 'file:///foo/bar'
+      @knife_ec2_create.validation_key_path.should eq('/foo/bar')
+    end
+
+    it 'returns a path to a tmp file when presented with a URI for the ' \
+      'validation key' do
+      Chef::Config[:knife][:validation_key_url] = @validation_key_url
+
+      @knife_ec2_create.stub_chain(:validation_key_tmpfile, :path).and_return(@validation_key_file)
+
+      @knife_ec2_create.validation_key_path.should eq(@validation_key_file)
+    end
+
     it "disallows security group names when using a VPC" do
       @knife_ec2_create.config[:subnet_id] = 'subnet-1a2b3c4d'
       @knife_ec2_create.config[:security_group_ids] = 'sg-aabbccdd'

spec/unit/s3_source_spec.rb

@@ -0,0 +1,53 @@
+require File.expand_path('../../spec_helper', __FILE__)
+require 'fog'
+
+describe Chef::Knife::S3Source do
+  before(:each) do
+    @bucket_name = 'my.bucket'
+    @test_file_path = 'path/to/file.pem'
+    @test_file_content = "TEST CONTENT\n"
+
+    Fog.mock!
+
+    {
+      aws_access_key_id: 'aws_access_key_id',
+      aws_secret_access_key: 'aws_secret_access_key'
+    }.each do |key, value|
+      Chef::Config[:knife][key] = value
+    end
+
+    fog = Fog::Storage::AWS.new(
+      aws_access_key_id: 'aws_access_key_id',
+      aws_secret_access_key: 'aws_secret_access_key'
+    )
+    test_dir_obj = fog.directories.create('key' => @bucket_name)
+    test_file_obj = test_dir_obj.files.create('key' => @test_file_path)
+    test_file_obj.body = @test_file_content
+    test_file_obj.save
+
+    @s3_connection = double(Fog::Storage::AWS)
+    @s3_source = Chef::Knife::S3Source.new
+
+    @s3_source.url = "s3://#{@bucket_name}/#{@test_file_path}"
+  end
+
+  it 'converts URI to path with leading / removed' do
+    @s3_source.instance_eval { path }
+    @s3_source.instance_eval { path }.should eq(@test_file_path)
+  end
+
+  it 'correctly retrieves the bucket name from the URI' do
+    @s3_source.instance_eval { bucket }
+    @s3_source.instance_eval { bucket }.should eq(@bucket_name)
+  end
+
+  it 'gets back the correct bucket contents' do
+    @s3_source.body.should eq(@test_file_content)
+  end
+
+  it 'gets back a bucket object with bucket_obj' do
+    @s3_source.instance_eval { bucket_obj }
+    @s3_source.instance_eval { bucket_obj }.should
+      be_kind_of(Fog::Storage::AWS::Directory)
+  end
+end