File: //snap/google-cloud-cli/394/lib/googlecloudsdk/command_lib/ai/docker/utils.py
# -*- coding: utf-8 -*- #
# Copyright 2021 Google LLC. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Common utilities to operate with Docker."""
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
import collections
import datetime
import re
from googlecloudsdk.command_lib.ai import errors
from googlecloudsdk.command_lib.ai.custom_jobs import local_util
from googlecloudsdk.core import log
_MAX_REPOSITORY_LENGTH = 255
_MAX_TAG_LENGTH = 128
_AUTONAME_PREFIX = "cloudai-autogenerated"
_DEFAULT_IMAGE_NAME = "unnamed"
_DEFAULT_REPO_REGION = "us"
Package = collections.namedtuple("Package",
["script", "package_path", "python_module"])
Image = collections.namedtuple("Image",
["name", "default_home", "default_workdir"])
def _ParseRepositoryTag(image_name):
"""Parses out the repository and tag from a Docker image name.
Args:
image_name: (str) The full name of an image, expected to be in a format of
"repository[:tag]"
Returns:
A (repository, tag) tuple representing the parsed result.
None repository means the image name is invalid; tag may be None if it isn't
present in the given image name.
"""
if image_name.count(":") > 2:
return None, None
parts = image_name.rsplit(":", 1)
if len(parts) == 2 and "/" not in parts[1]:
return tuple(parts)
return image_name, None
def _ParseRepositoryHost(repository_name):
"""Parses a repository to an optional hostname and a list of path compoentes.
Args:
repository_name: (str) A name made up of slash-separated path name
components, optionally prefixed by a registry hostname.
Returns:
A (hostname, components) tuple representing the parsed result.
The hostname will be None if it isn't present; the components is a list of
each slash-separated part in the given repository name.
"""
components = repository_name.split("/")
if len(components) == 1:
return None, components
if "." in components[0] or ":" in components[0]:
# components[0] is regarded as a hostname
return components[0], components[1:]
return None, components
def _ParseHostPort(host):
"""Parses a registry hostname to a list of components and an optional port.
Args:
host: (str) The registry hostname supposed to comply with standard DNS
rules, optionally be followed by a port number in the format like ":8080".
Returns:
A (hostcomponents, port) tuple representing the parsed result.
The hostcomponents contains each dot-seperated component in the given
hostname; port may be None if it isn't present.
"""
parts = host.rsplit(":", 1)
hostcomponents = parts[0].split(".")
port = parts[1] if len(parts) == 2 else None
return hostcomponents, port
def ValidateRepositoryAndTag(image_name):
r"""Validate the given image name is a valid repository/tag reference.
As explained in
https://docs.docker.com/engine/reference/commandline/tag/#extended-description,
a valid repository/tag reference should following the below pattern:
reference := name [ ":" tag ]
name := [hostname '/'] component ['/' component]*
hostname := hostcomponent ['.' hostcomponent]* [':' port-number]
hostcomponent := /([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])/
port-number := /[0-9]+/
component := alpha-numeric [separator alpha-numeric]*
alpha-numeric := /[a-z0-9]+/
separator := /[_.]|__|[-]*/
tag := /[\w][\w.-]{0,127}/
Args:
image_name: (str) Full name of a Docker image.
Raises:
ValueError if the image name is not valid.
"""
repository, tag = _ParseRepositoryTag(image_name)
if repository is None:
raise ValueError("Unable to parse repository and tag.")
if len(repository) > _MAX_REPOSITORY_LENGTH:
raise ValueError(
"Repository name must not be more than {} characters.".format(
_MAX_REPOSITORY_LENGTH))
hostname, path_components = _ParseRepositoryHost(repository)
if hostname:
hostcomponents, port = _ParseHostPort(hostname)
hostcomponent_regex = r"^(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])$"
for hostcomponent in hostcomponents:
if re.match(hostcomponent_regex, hostcomponent) is None:
raise ValueError(
"Invalid hostname/port \"{}\" in repository name.".format(hostname))
port_regex = r"^[0-9]+$"
if port and re.match(port_regex, port) is None:
raise ValueError(
"Invalid hostname/port \"{}\" in repository name.".format(hostname))
for component in path_components:
if not component:
raise ValueError("Empty path component in repository name.")
component_regex = r"^[a-z0-9]+(?:(?:[._]|__|[-]*)[a-z0-9]+)*$"
if re.match(component_regex, component) is None:
raise ValueError(
"Invalid path component \"{}\" in repository name.".format(component))
if tag:
if len(tag) > _MAX_TAG_LENGTH:
raise ValueError("Tag name must not be more than {} characters.".format(
_MAX_TAG_LENGTH))
tag_regex = r"^[\w][\w.-]{0,127}$"
if re.match(tag_regex, tag) is None:
raise ValueError("Invalid tag.")
def GenerateImageName(base_name=None, project=None, region=None, is_gcr=False):
"""Generate a name for the Docker image built by AI platform gcloud."""
sanitized_name = _SanitizeRepositoryName(base_name or _DEFAULT_IMAGE_NAME)
# Use the current timestamp as the tag.
tag = datetime.datetime.now().strftime("%Y%m%d.%H.%M.%S.%f")
image_name = "{}/{}:{}".format(_AUTONAME_PREFIX, sanitized_name, tag)
if project:
if is_gcr:
repository = "gcr.io"
else:
region_prefix = region or _DEFAULT_REPO_REGION
repository = "{}-docker.pkg.dev".format(region_prefix)
return "{}/{}/{}".format(repository, project.replace(":", "/"), image_name)
return image_name
def _SanitizeRepositoryName(name):
"""Sanitizes the given name to make it valid as an image repository.
As explained in
https://docs.docker.com/engine/reference/commandline/tag/#extended-description,
Valid name may contain only lowercase letters, digits and separators.
A separator is defined as a period, one or two underscores, or one or more
dashes. A name component may not start or end with a separator.
This method will replace the illegal characters in the given name and strip
starting and ending separator characters.
Args:
name: str, the name to sanitize.
Returns:
A sanitized name.
"""
return re.sub("[._][._]+|[^a-z0-9._-]+", ".", name.lower()).strip("._-")
def ExecuteDockerCommand(command):
"""Executes Docker CLI commands in subprocess.
Just calls local_util.ExecuteCommand(cmd,...) and raises error for non-zero
exit code.
Args:
command: (List[str]) Strings to send in as the command.
Raises:
ValueError: The input command is not a docker command.
DockerError: An error occurred when executing the given docker command.
"""
command_str = " ".join(command)
if not command_str.startswith("docker"):
raise ValueError("`{}` is not a Docker command".format("docker"))
log.info("Running command: {}".format(command_str))
return_code = local_util.ExecuteCommand(command)
if return_code != 0:
error_msg = """
Docker failed with error code {code}.
Command: {cmd}
""".format(
code=return_code, cmd=command_str)
raise errors.DockerError(error_msg, command, return_code)