# Copyright 2017 datawire. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import base64, boto3, os, urllib2, hashlib
from tasks import task, TaskError, get, sh, Secret
[docs]class DockerImageBuilderError(TaskError):
report_traceback = False
pass
[docs]class DockerImageBuilder(object):
DOCKER = 'docker'
IMAGEBUILDER = 'imagebuilder'
[docs] @classmethod
def get_cmd_from_name(self, str):
if str == self.DOCKER:
def docker_build(directory, dockerfile, img, buildargs):
return ["docker", "build", directory, "-f", dockerfile, "-t", img] + buildargs
return docker_build
elif str == self.IMAGEBUILDER:
def imagebuilder_build(directory, dockerfile, img, buildargs):
return ["imagebuilder", "-f", dockerfile, "-t", img] + buildargs + [directory]
return imagebuilder_build
raise DockerImageBuilderError("No image builder named %s exists. Available builders are: %s" % (str, ", ".join([self.DOCKER, self.IMAGEBUILDER])))
[docs]def image(registry, namespace, name, version):
parts = (registry, namespace, "%s:%s" % (name, version))
return "/".join(p for p in parts if p)
[docs]class DockerBase(object):
def __init__(self):
self.image_cache = {}
self.logged_in = False
def _login(self):
if not self.logged_in:
self._do_login()
self.logged_in = True
@task()
def local_exists(self, name, version):
return bool(sh("docker", "images", "-q", self.image(name, version)).output)
@task()
def exists(self, name, version):
return self.remote_exists(name, version) or self.local_exists(name, version)
@task()
def needs_push(self, name, version):
return self.local_exists(name, version) and not self.remote_exists(name, version)
@task()
def pull(self, image):
self._login()
sh("docker", "pull", image)
@task()
def tag(self, source, name, version):
img = self.image(name, version)
sh("docker", "tag", source, img)
def _create_repo(self, name):
pass
@task()
def push(self, name, version):
self._login()
self._create_repo(name)
img = self.image(name, version)
self.image_cache.pop(img, None)
sh("docker", "push", img)
return img
@task()
def build(self, directory, dockerfile, name, version, args, builder=None):
args = args or {}
builder = builder or DockerImageBuilder.DOCKER
buildargs = []
for k, v in args.items():
buildargs.append("--build-arg")
buildargs.append("%s=%s" % (k, v))
img = self.image(name, version)
cmd = DockerImageBuilder.get_cmd_from_name(builder)
sh(*cmd(directory, dockerfile, img, buildargs))
return img
[docs] def get_changes(self, dockerfile):
entrypoint = None
cmd = None
with open(dockerfile) as f:
for line in f:
parts = line.split()
if parts and parts[0].lower() == "cmd":
cmd = line
elif parts and parts[0].lower() == "entrypoint":
entrypoint = line
return (entrypoint or 'ENTRYPOINT []', cmd or 'CMD []')
[docs] def builder_hash(self, dockerfile, args):
result = hashlib.sha1()
with open(dockerfile) as fd:
result.update(fd.read())
result.update("--")
for a in sorted(args.keys()):
result.update(a)
result.update("--")
result.update(args[a])
result.update("--")
return result.hexdigest()
[docs] def builder_prefix(self, name):
return "forge_%s" % name
[docs] def find_builders(self, name):
builder_prefix = self.builder_prefix(name)
containers = sh("docker", "ps", "-qaf", "name=%s" % builder_prefix, "--format", "{{.ID}} {{.Names}}")
for line in containers.output.splitlines():
id, builder_name = line.split()
yield id, builder_name
@task()
def builder(self, directory, dockerfile, name, version, args, builder=None):
# We hash the buildargs and Dockerfile so that we reconstruct
# the builder container if anything changes. This might want
# to be extended to cover other files the Dockerfile
# references somehow at some point. (Maybe we could use the
# spec stuff we use in .forgeignore?)
builder_name = "%s_%s" % (self.builder_prefix(name), self.builder_hash(dockerfile, args))
cid = None
for id, bname in self.find_builders(name):
if bname == builder_name:
cid = id
else:
Builder(self, id).kill()
if not cid:
image = self.build(directory, dockerfile, name, version, args, builder=None)
cid = sh("docker", "run", "--rm", "--name", builder_name, "-dit", "--entrypoint", "/bin/sh",
image).output.strip()
return Builder(self, cid, self.get_changes(dockerfile))
@task()
def clean(self, name):
for id, bname in self.find_builders(name):
Builder(self, id).kill()
@task()
def validate(self, name="forge_test"):
test_image = os.environ.get("FORGE_SETUP_IMAGE", "registry.hub.docker.com/datawire/forge-setup-test:1")
self.pull(test_image)
version = "dummy"
self.tag(test_image, name, version)
self.push(name, version)
assert self.remote_exists(name, version)
@task()
def run(self, name, version, cmd, *args):
return sh("docker", "run", "--rm", "-it", "--entrypoint", cmd, self.image(name, version), *args)
[docs]class Builder(object):
def __init__(self, docker, cid, changes=()):
self.docker = docker
self.cid = cid
self.changes = changes
[docs] def run(self, *args):
# XXX: for some reason when we put a -t here it messes up the
# terminal output
return sh("docker", "exec", "-i", self.cid, *args)
[docs] def cp(self, source, target):
return sh("docker", "cp", source, "{0}:{1}".format(self.cid, target))
[docs] def commit(self, name, version):
args = []
for change in self.changes:
args.append("-c")
args.append(change)
args.extend((self.cid, self.docker.image(name, version)))
return sh("docker", "commit", *args)
[docs] def kill(self):
sh("docker", "kill", self.cid, expected=(0, 1))
import json, base64
[docs]class Docker(DockerBase):
def __init__(self, registry, namespace, user, password, verify=True):
DockerBase.__init__(self)
self.registry = registry
self.namespace = namespace
self.user = user
self.password = password
self.verify = verify
self._run_login = bool(self.user)
if not self.user:
docker_config = os.path.join(os.environ.get("HOME"), ".docker/config.json")
if os.path.exists(docker_config):
with open(docker_config) as fd:
cfg = json.load(fd)
auths = cfg.get("auths", {})
auth = auths.get(self.registry, {}).get("auth")
if auth:
self.user, self.password = base64.decodestring(auth).split(":")
if not self._run_login and not self.user:
raise TaskError("unable to locate docker credentials, please run `docker login %s`" % self.registry)
@task()
def image(self, name, version):
return image(self.registry, self.namespace, name, version)
def _do_login(self):
if self._run_login:
sh("docker", "login", "-u", self.user, "-p", Secret(self.password), self.registry)
@task()
def registry_get(self, api):
url = "https://%s/v2/%s" % (self.registry, api)
response = get(url, auth=(self.user, self.password),
headers={"Accept": 'application/vnd.docker.distribution.manifest.v2+json'},
verify=self.verify)
if response.status_code == 401:
challenge = response.headers['Www-Authenticate']
if challenge.startswith("Bearer "):
challenge = challenge[7:]
opts = urllib2.parse_keqv_list(urllib2.parse_http_list(challenge))
authresp = get("{realm}?service={service}&scope={scope}".format(**opts), auth=(self.user, self.password),
verify=self.verify)
if authresp.ok:
token = authresp.json()['token']
response = get(url, headers={'Authorization': 'Bearer %s' % token},
verify=self.verify)
else:
raise TaskError("problem authenticating with docker registry: [%s] %s" % (authresp.status_code,
authresp.content))
return response
@task()
def repo_get(self, name, api):
return self.registry_get("%s/%s/%s" % (self.namespace, name, api))
@task()
def remote_exists(self, name, version):
self._login()
img = self.image(name, version)
if img in self.image_cache:
return self.image_cache[img]
response = self.repo_get(name, "manifests/%s" % version)
result = response.json()
# v1 and v2 manifest schemas look a bit different
if 'fsLayers' in result or 'layers' in result:
self.image_cache[img] = True
return True
elif 'errors' in result and result['errors']:
if result['errors'][0]['code'] in ('MANIFEST_UNKNOWN', 'NAME_UNKNOWN'):
self.image_cache[img] = False
return False
raise TaskError(response.content)
[docs]class GCRDocker(Docker):
def __init__(self, url, project, key):
Docker.__init__(self, url, project, "_json_key" if key else "_token", key)
def _do_login(self):
if self.user == "_token":
self.password = sh("gcloud", "auth", "print-access-token",
output_transform = lambda x: "<OUTPUT_ELIDED>").output.strip()
Docker._do_login(self)
def _get_account():
sts = boto3.client('sts')
return sts.get_caller_identity()["Account"]
def _get_region():
return boto3.Session().region_name
[docs]class ECRDocker(DockerBase):
def __init__(self, account=None, region=None, aws_access_key_id=None, aws_secret_access_key=None):
DockerBase.__init__(self)
self.account = account or _get_account()
self.region = region or _get_region()
kwargs = {}
if aws_access_key_id: kwargs['aws_access_key_id'] = aws_access_key_id
if aws_secret_access_key: kwargs['aws_secret_access_key'] = aws_secret_access_key
self.ecr = boto3.client('ecr', self.region, **kwargs)
self.url = "{}.dkr.ecr.{}.amazonaws.com".format(self.account, self.region)
@property
def registry(self):
return self.url
@property
def namespace(self):
return None
def _do_login(self):
response = self.ecr.get_authorization_token(registryIds=[self.account])
data = response['authorizationData'][0]
token = data['authorizationToken']
user, password = base64.decodestring(token).split(":")
proxy = data['proxyEndpoint']
sh("docker", "login", "-u", user, "-p", Secret(password), proxy)
@task()
def image(self, name, version):
return "{}/{}:{}".format(self.url, name, version)
#return image(self.registry, self.namespace, name, version)
def _create_repo(self, name):
try:
self.ecr.create_repository(repositoryName=name)
task.info('repository {} created'.format(name))
except self.ecr.exceptions.RepositoryAlreadyExistsException, e:
task.info('repository {} already exists'.format(name))
@task()
def remote_exists(self, name, version):
try:
task.info('checking for remote version: %r' % version)
response = self.ecr.describe_images(registryId=self.account,
repositoryName=name,
imageIds=[{'imageTag': version}])
tags = set([t for id in response['imageDetails'] for t in id['imageTags']])
return version in tags
except self.ecr.exceptions.ImageNotFoundException, e:
return False
except self.ecr.exceptions.RepositoryNotFoundException, e:
return False
[docs]class LocalDocker(DockerBase):
[docs] def image(self, name, version):
return "{}:{}".format(name, version)
[docs] def remote_exists(self, name, version):
return False
[docs] def needs_push(self, name, version):
return False