File: //snap/google-cloud-cli/396/lib/surface/compute/images/import.py
# -*- coding: utf-8 -*- #
# Copyright 2017 Google LLC. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Import image command."""
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
import abc
import enum
import os.path
import string
import uuid
from googlecloudsdk.api_lib.compute import base_classes
from googlecloudsdk.api_lib.compute import daisy_utils
from googlecloudsdk.api_lib.compute import image_utils
from googlecloudsdk.api_lib.compute import utils
from googlecloudsdk.api_lib.storage import storage_api
from googlecloudsdk.api_lib.storage import storage_util
from googlecloudsdk.calliope import base
from googlecloudsdk.calliope import exceptions
from googlecloudsdk.command_lib.compute.images import flags
from googlecloudsdk.command_lib.compute.images import os_choices
from googlecloudsdk.core import log
from googlecloudsdk.core import properties
from googlecloudsdk.core import resources
from googlecloudsdk.core.console import progress_tracker
import six
_WORKFLOWS_URL = (
'https://github.com/GoogleCloudPlatform/compute-image-import/'
'tree/main/daisy_workflows/image_import'
)
_OUTPUT_FILTER = [
'[Daisy',
'[import-',
'[onestep-',
'starting build',
' import',
'ERROR',
]
class CloudProvider(enum.Enum):
UNKNOWN = 0
AWS = 1
def _HasAwsArgs(args):
return (
args.aws_access_key_id
or args.aws_secret_access_key
or args.aws_session_token
or args.aws_region
or args.aws_ami_id
or args.aws_ami_export_location
or args.aws_source_ami_file_path
)
def _HasExternalCloudProvider(args):
return _GetExternalCloudProvider(args) != CloudProvider.UNKNOWN
def _GetExternalCloudProvider(args):
if _HasAwsArgs(args):
return CloudProvider.AWS
return CloudProvider.UNKNOWN
def _AppendTranslateWorkflowArg(args, import_args):
if args.os:
daisy_utils.AppendArg(import_args, 'os', args.os)
daisy_utils.AppendArg(
import_args, 'custom_translate_workflow', args.custom_workflow
)
def _AppendAwsArgs(args, import_args):
"""Appends args related to AWS image import."""
daisy_utils.AppendArg(
import_args, 'aws_access_key_id', args.aws_access_key_id
)
daisy_utils.AppendArg(
import_args, 'aws_secret_access_key', args.aws_secret_access_key
)
daisy_utils.AppendArg(
import_args, 'aws_session_token', args.aws_session_token
)
daisy_utils.AppendArg(import_args, 'aws_region', args.aws_region)
if args.aws_ami_id:
daisy_utils.AppendArg(import_args, 'aws_ami_id', args.aws_ami_id)
if args.aws_ami_export_location:
daisy_utils.AppendArg(
import_args, 'aws_ami_export_location', args.aws_ami_export_location
)
if args.aws_source_ami_file_path:
daisy_utils.AppendArg(
import_args, 'aws_source_ami_file_path', args.aws_source_ami_file_path
)
def _CheckImageName(image_name):
"""Checks for a valid GCE image name."""
name_message = (
'Name must start with a lowercase letter followed by up to '
'63 lowercase letters, numbers, or hyphens, and cannot end '
'with a hyphen.'
)
name_ok = True
valid_chars = string.digits + string.ascii_lowercase + '-'
if len(image_name) > 64:
name_ok = False
elif image_name[0] not in string.ascii_lowercase:
name_ok = False
elif not all(char in valid_chars for char in image_name):
name_ok = False
elif image_name[-1] == '-':
name_ok = False
if not name_ok:
raise exceptions.InvalidArgumentException('IMAGE_NAME', name_message)
def _CheckForExistingImage(
image_name, compute_holder, arg_name='IMAGE_NAME', expect_to_exist=False
):
"""Check if image already exists."""
# Don't perform a check for image name used in E2E test as passing an invalid
# name to the backend is currently the only way to perform a quick sanity E2E
# check on the backend. Alternative is not to perform the check on whether
# image exists at all which would lead to worse customer experience. No
# security issue with this as the backend does the same check on whether an
# image exists or not.
expect_to_exist_image_name_exclusions = ['sample-image-123']
if expect_to_exist and image_name in expect_to_exist_image_name_exclusions:
return
image_ref = resources.REGISTRY.Parse(
image_name,
collection='compute.images',
params={'project': properties.VALUES.core.project.GetOrFail},
)
image_expander = image_utils.ImageExpander(
compute_holder.client, compute_holder.resources
)
try:
_ = image_expander.GetImage(image_ref)
image_exists = True
except utils.ImageNotFoundError:
image_exists = False
if not expect_to_exist and image_exists:
message = 'The image [{0}] already exists.'.format(image_name)
raise exceptions.InvalidArgumentException(arg_name, message)
elif expect_to_exist and not image_exists:
message = 'The image [{0}] does not exist.'.format(image_name)
raise exceptions.InvalidArgumentException(arg_name, message)
@base.ReleaseTracks(base.ReleaseTrack.GA)
@base.Deprecate(
is_removed=False,
warning=(
'This command is being deprecated. Instead, use the `gcloud migration'
' vms image-imports` command. For more information, see https://'
'cloud.google.com/migrate/virtual-machines/docs/5.0/migrate/'
'image_import.'
),
error=(
'This command hash been deprecated. Instead, use the `gcloud migration'
' vms image-imports` command. For more information, see https://'
'cloud.google.com/migrate/virtual-machines/docs/5.0/migrate/'
'image_import.'
),
)
@base.DefaultUniverseOnly
class Import(base.CreateCommand):
"""Import an image into Compute Engine."""
_OS_CHOICES = os_choices.OS_CHOICES_IMAGE_IMPORT_GA
def __init__(self, *args, **kwargs):
self.storage_client = storage_api.StorageClient()
super(Import, self).__init__(*args, **kwargs)
@classmethod
def Args(cls, parser):
compute_holder = cls._GetComputeApiHolder(no_http=True)
compute_client = compute_holder.client
messages = compute_client.messages
Import.DISK_IMAGE_ARG = flags.MakeDiskImageArg()
Import.DISK_IMAGE_ARG.AddArgument(parser, operation_type='create')
flags.compute_flags.AddZoneFlag(
parser,
'image',
'import',
help_text=(
'Zone to use when importing the image. When you import '
'an image, the import tool creates and uses temporary VMs '
'in your project for the import process. Use this flag to '
'specify the zone to use for these temporary VMs.'
),
)
source = parser.add_mutually_exclusive_group(required=True, sort_args=False)
import_from_local_or_gcs = source.add_mutually_exclusive_group(
help=(
'Image import from local file, Cloud Storage or '
'Compute Engine image.'
)
)
import_from_local_or_gcs.add_argument(
'--source-file',
help=("""A local file, or the Cloud Storage URI of the virtual
disk file to import. For example: ``gs://my-bucket/my-image.vmdk''
or ``./my-local-image.vmdk''. For more information about Cloud
Storage URIs, see
https://cloud.google.com/storage/docs/request-endpoints#json-api.
"""),
)
flags.SOURCE_IMAGE_ARG.AddArgument(
import_from_local_or_gcs, operation_type='import'
)
import_from_aws = source.add_group(help='Image import from AWS.')
daisy_utils.AddAWSImageImportSourceArgs(import_from_aws)
image_utils.AddGuestOsFeaturesArgForImport(parser, messages)
workflow = parser.add_mutually_exclusive_group()
os_group = workflow.add_group()
daisy_utils.AddByolArg(os_group)
os_group.add_argument(
'--os',
choices=sorted(cls._OS_CHOICES),
help='Specifies the OS of the disk image being imported.',
)
workflow.add_argument(
'--data-disk',
help=(
'Specifies that the disk has no bootable OS installed on it. '
'Imports the disk without making it bootable or installing '
'Google tools on it.'
),
action='store_true',
)
workflow.add_argument(
'--custom-workflow',
help=(
"""\
Specifies a custom workflow to use for image translation. Workflow
should be relative to the image_import directory here: []({0}).
For example: `debian/translate_debian_9.wf.json'""".format(
_WORKFLOWS_URL
)
),
hidden=True,
)
daisy_utils.AddCommonDaisyArgs(
parser,
operation='an import',
extra_timeout_help=("""
If you are importing a large image that takes longer than 24 hours to
import, either use the RAW disk format to reduce the time needed for
converting the image, or split the data into several smaller images.
"""),
)
parser.add_argument(
'--guest-environment',
action='store_true',
default=True,
help=(
'Installs the guest environment on the image.'
' See '
'https://cloud.google.com/compute/docs/images/guest-environment.'
),
)
parser.add_argument(
'--network',
help=(
'Name of the network in your project to use for the image '
'import. When you import an image, the import tool creates and '
'uses temporary VMs in your project for the import process. Use '
'this flag to specify the network to use for these temporary VMs.'
),
)
parser.add_argument(
'--subnet',
help=("""\
Name of the subnetwork in your project to use for the image import. When
you import an image, the import tool creates and uses temporary VMs in
your project for the import process. Use this flag to specify the
subnetwork to use for these temporary VMs.
* If the network resource is in legacy mode, do not provide this
property.
* If the network is in auto subnet mode, specifying the subnetwork is
optional.
* If the network is in custom subnet mode, then this field must be
specified.
"""),
)
parser.add_argument(
'--family', help='Family to set for the imported image.'
)
parser.add_argument(
'--description', help='Description to set for the imported image.'
)
parser.display_info.AddCacheUpdater(flags.ImagesCompleter)
parser.add_argument(
'--storage-location',
help="""\
Specifies a Cloud Storage location, either regional or multi-regional,
where image content is to be stored. If not specified, the multi-region
location closest to the source is chosen automatically.
""",
)
parser.add_argument(
'--sysprep-windows',
action='store_true',
hidden=True,
help='Whether to generalize the image using Windows Sysprep.',
)
daisy_utils.AddNoAddressArg(
parser,
'image import',
'https://cloud.google.com/compute/docs/import/importing-virtual-disks#no-external-ip',
)
daisy_utils.AddComputeServiceAccountArg(
parser,
'image import',
daisy_utils.IMPORT_ROLES_FOR_COMPUTE_SERVICE_ACCOUNT,
)
daisy_utils.AddCloudBuildServiceAccountArg(
parser,
'image import',
daisy_utils.IMPORT_ROLES_FOR_CLOUDBUILD_SERVICE_ACCOUNT,
)
parser.add_argument(
'--cmd-deprecated',
action='store_true',
required=True,
help="""
The command you're using is deprecated and will be removed by December 31,
2025. We recommend using `gcloud compute migration image-imports` instead.
See our official documentation for more information.
https://cloud.google.com/migrate/virtual-machines/docs/5.0/migrate/image_import.
""",
)
@classmethod
def _GetComputeApiHolder(cls, no_http=False):
return base_classes.ComputeApiHolder(cls.ReleaseTrack(), no_http)
def Run(self, args):
compute_holder = self._GetComputeApiHolder()
# Fail early if the requested image name is invalid or already exists.
_CheckImageName(args.image_name)
_CheckForExistingImage(args.image_name, compute_holder)
stager = self._CreateImportStager(args, compute_holder)
import_metadata = stager.Stage()
log.warning('Importing image. This may take up to 2 hours.')
tags = ['gce-daisy-image-import']
return self._RunImageImport(args, import_metadata, tags, _OUTPUT_FILTER)
def _RunImageImport(self, args, import_args, tags, output_filter):
image_tag = daisy_utils.GetDefaultBuilderVersion()
if hasattr(args, 'docker_image_tag'):
image_tag = args.docker_image_tag
if _HasExternalCloudProvider(args):
return daisy_utils.RunOnestepImageImport(
args,
import_args,
tags,
_OUTPUT_FILTER,
release_track=self.ReleaseTrack().id.lower()
if self.ReleaseTrack()
else None,
docker_image_tag=image_tag,
)
return daisy_utils.RunImageImport(
args,
import_args,
tags,
_OUTPUT_FILTER,
release_track=self.ReleaseTrack().id.lower()
if self.ReleaseTrack()
else None,
docker_image_tag=image_tag,
)
def _CreateImportStager(self, args, compute_holder):
if _HasExternalCloudProvider(args):
return ImportFromExternalCloudProviderStager(
self.storage_client, compute_holder, args
)
if args.source_image:
return ImportFromImageStager(self.storage_client, compute_holder, args)
if daisy_utils.IsLocalFile(args.source_file):
return ImportFromLocalFileStager(
self.storage_client, compute_holder, args
)
try:
gcs_uri = daisy_utils.MakeGcsObjectUri(args.source_file)
except storage_util.InvalidObjectNameError:
raise exceptions.InvalidArgumentException(
'source-file', 'must be a path to an object in Google Cloud Storage'
)
else:
return ImportFromGSFileStager(
self.storage_client, compute_holder, args, gcs_uri
)
@six.add_metaclass(abc.ABCMeta)
class BaseImportStager(object):
"""Base class for image import stager.
An abstract class which is responsible for preparing import parameters, such
as Daisy parameters and workflow, as well as creating Daisy scratch bucket in
the appropriate location.
"""
def __init__(self, storage_client, compute_holder, args):
self.storage_client = storage_client
self.compute_holder = compute_holder
self.args = args
self.daisy_bucket = self.GetAndCreateDaisyBucket()
def Stage(self):
"""Prepares for import args.
It supports running new import wrapper (gce_vm_image_import).
Returns:
import_args - array of strings, import args.
"""
import_args = []
messages = self.compute_holder.client.messages
daisy_utils.AppendArg(
import_args, 'zone', properties.VALUES.compute.zone.Get()
)
if self.args.storage_location:
daisy_utils.AppendArg(
import_args, 'storage_location', self.args.storage_location
)
daisy_utils.AppendArg(
import_args,
'scratch_bucket_gcs_path',
'gs://{0}/'.format(self.daisy_bucket),
)
daisy_utils.AppendArg(
import_args,
'timeout',
'{}s'.format(daisy_utils.GetDaisyTimeout(self.args)),
)
daisy_utils.AppendArg(import_args, 'client_id', 'gcloud')
daisy_utils.AppendArg(import_args, 'image_name', self.args.image_name)
daisy_utils.AppendBoolArg(
import_args, 'no_guest_environment', not self.args.guest_environment
)
daisy_utils.AppendNetworkAndSubnetArgs(self.args, import_args)
daisy_utils.AppendArg(import_args, 'description', self.args.description)
daisy_utils.AppendArg(import_args, 'family', self.args.family)
if 'byol' in self.args:
daisy_utils.AppendBoolArg(import_args, 'byol', self.args.byol)
# The value of the attribute 'guest_os_features' can be can be a list, None,
# or the attribute may not be present at all.
# We treat the case when it is None or when it is not present as if the list
# of features is empty. We need to use the trailing `or ()` rather than
# give () as a default value to getattr() to handle the case where
# args.guest_os_features is present, but it is None.
guest_os_features = getattr(self.args, 'guest_os_features', None) or ()
uefi_compatible = (
messages.GuestOsFeature.TypeValueValuesEnum.UEFI_COMPATIBLE.name
in guest_os_features
)
if uefi_compatible:
daisy_utils.AppendBoolArg(import_args, 'uefi_compatible', True)
if 'sysprep_windows' in self.args:
daisy_utils.AppendBoolArg(
import_args, 'sysprep_windows', self.args.sysprep_windows
)
if 'no_address' in self.args:
daisy_utils.AppendBoolArg(
import_args, 'no_external_ip', self.args.no_address
)
if 'compute_service_account' in self.args:
daisy_utils.AppendArg(
import_args,
'compute_service_account',
self.args.compute_service_account,
)
return import_args
def GetAndCreateDaisyBucket(self):
return daisy_utils.CreateDaisyBucketInProject(
self.GetBucketLocation(),
self.storage_client,
enable_uniform_level_access=True,
soft_delete_duration=0,
)
def GetBucketLocation(self):
if self.args.storage_location:
return self.args.storage_location
return None
class ImportFromExternalCloudProviderStager(BaseImportStager):
"""Image import stager from an external cloud provider."""
def Stage(self):
import_args = []
_AppendAwsArgs(self.args, import_args)
_AppendTranslateWorkflowArg(self.args, import_args)
import_args.extend(
super(ImportFromExternalCloudProviderStager, self).Stage()
)
return import_args
def GetBucketLocation(self):
if self.args.zone:
return daisy_utils.GetRegionFromZone(self.args.zone)
return super(
ImportFromExternalCloudProviderStager, self
).GetBucketLocation()
class ImportFromImageStager(BaseImportStager):
"""Image import stager from an existing image."""
def Stage(self):
_CheckForExistingImage(
self.args.source_image,
self.compute_holder,
arg_name='source-image',
expect_to_exist=True,
)
import_args = []
daisy_utils.AppendArg(import_args, 'source_image', self.args.source_image)
if self.args.data_disk:
daisy_utils.AppendBoolArg(import_args, 'data_disk', self.args.data_disk)
else:
_AppendTranslateWorkflowArg(self.args, import_args)
import_args.extend(super(ImportFromImageStager, self).Stage())
return import_args
def _GetSourceImage(self):
ref = resources.REGISTRY.Parse(
self.args.source_image,
collection='compute.images',
params={'project': properties.VALUES.core.project.GetOrFail},
)
# source_name should be of the form 'global/images/image-name'.
source_name = ref.RelativeName()[len(ref.Parent().RelativeName() + '/') :]
return source_name
def GetBucketLocation(self):
if self.args.zone:
return daisy_utils.GetRegionFromZone(self.args.zone)
return super(ImportFromImageStager, self).GetBucketLocation()
class BaseImportFromFileStager(BaseImportStager):
"""Abstract image import stager for import from a file."""
def Stage(self):
self._FileStage()
import_args = []
# Import and (maybe) translate from the scratch bucket.
daisy_utils.AppendArg(import_args, 'source_file', self.gcs_uri)
if self.args.data_disk:
daisy_utils.AppendBoolArg(import_args, 'data_disk', self.args.data_disk)
else:
_AppendTranslateWorkflowArg(self.args, import_args)
import_args.extend(super(BaseImportFromFileStager, self).Stage())
return import_args
def _FileStage(self):
"""Prepare image file for importing."""
# If the file is an OVA file, print a warning.
if self.args.source_file.lower().endswith('.ova'):
log.warning(
'The specified input file may contain more than one virtual disk. '
'Only the first vmdk disk will be imported. To import a .ova'
"completely, please try 'gcloud compute instances import'"
'instead.'
)
elif self.args.source_file.lower().endswith(
'.tar.gz'
) or self.args.source_file.lower().endswith('.tgz'):
raise exceptions.BadFileException(
'`gcloud compute images import` does not support compressed '
'archives. Please extract your image and try again.\n If you got '
'this file by exporting an image from Compute Engine (e.g., by '
'using `gcloud compute images export`) then you can instead use '
'`gcloud compute images create` to create your image from your '
'.tar.gz file.'
)
self.gcs_uri = self._CopySourceFileToScratchBucket()
@abc.abstractmethod
def _CopySourceFileToScratchBucket(self):
raise NotImplementedError
class ImportFromLocalFileStager(BaseImportFromFileStager):
"""Image import stager from a local file."""
def _CopySourceFileToScratchBucket(self):
return self._UploadToGcs(
self.args.async_, self.args.source_file, self.daisy_bucket, uuid.uuid4()
)
def _UploadToGcs(self, is_async, local_path, daisy_bucket, image_uuid):
"""Uploads a local file to GCS. Returns the gs:// URI to that file."""
file_name = os.path.basename(local_path).replace(' ', '-')
dest_path = 'gs://{0}/tmpimage/{1}-{2}'.format(
daisy_bucket, image_uuid, file_name
)
if is_async:
log.status.Print(
'Async: After upload is complete, your image will be '
'imported from Cloud Storage asynchronously.'
)
with progress_tracker.ProgressTracker(
'Copying [{0}] to [{1}]'.format(local_path, dest_path)
):
return self._UploadToGcsStorageApi(local_path, dest_path)
def _UploadToGcsStorageApi(self, local_path, dest_path):
"""Uploads a local file to Cloud Storage using the gcloud storage api client."""
dest_object = storage_util.ObjectReference.FromUrl(dest_path)
self.storage_client.CopyFileToGCS(local_path, dest_object)
return dest_path
class ImportFromGSFileStager(BaseImportFromFileStager):
"""Image import stager from a file in Cloud Storage."""
def __init__(self, storage_client, compute_holder, args, gcs_uri):
self.source_file_gcs_uri = gcs_uri
super(ImportFromGSFileStager, self).__init__(
storage_client, compute_holder, args
)
def GetBucketLocation(self):
return self.storage_client.GetBucketLocationForFile(
self.source_file_gcs_uri
)
def _CopySourceFileToScratchBucket(self):
image_file = os.path.basename(self.source_file_gcs_uri)
dest_uri = 'gs://{0}/tmpimage/{1}-{2}'.format(
self.daisy_bucket, uuid.uuid4(), image_file
)
src_object = resources.REGISTRY.Parse(
self.source_file_gcs_uri, collection='storage.objects'
)
dest_object = resources.REGISTRY.Parse(
dest_uri, collection='storage.objects'
)
with progress_tracker.ProgressTracker(
'Copying [{0}] to [{1}]'.format(self.source_file_gcs_uri, dest_uri)
):
self.storage_client.Rewrite(src_object, dest_object)
return dest_uri
@base.ReleaseTracks(base.ReleaseTrack.BETA)
class ImportBeta(Import):
"""Import an image into Compute Engine for beta releases."""
_OS_CHOICES = os_choices.OS_CHOICES_IMAGE_IMPORT_BETA
@classmethod
def Args(cls, parser):
super(ImportBeta, cls).Args(parser)
daisy_utils.AddExtraCommonDaisyArgs(parser)
@base.ReleaseTracks(base.ReleaseTrack.ALPHA)
class ImportAlpha(ImportBeta):
"""Import an image into Compute Engine for alpha releases."""
_OS_CHOICES = os_choices.OS_CHOICES_IMAGE_IMPORT_ALPHA
Import.detailed_help = {
'brief': 'Import an image into Compute Engine',
'DESCRIPTION': """
*{command}* imports Virtual Disk images, such as VMWare VMDK files
and VHD files, into Compute Engine.
Importing images involves four steps:
* Upload the virtual disk file to Cloud Storage.
* Import the image to Compute Engine.
* Detect the OS and bootloader contained on the disk.
* Translate the image to make a bootable image.
This command performs all four of these steps as required,
depending on the input arguments specified.
Before importing an image, set up access to Cloud Storage and grant
required roles to the user accounts and service accounts. For more
information, see [](https://cloud.google.com/compute/docs/import/requirements-export-import-images).
To override the detected OS, specify the `--os` flag.
You can omit the translation step using the `--data-disk` flag.
If you exported your disk from Compute Engine then you don't
need to re-import it. Instead, use `{parent_command} create`
to create more images from the disk.
Files stored on Cloud Storage and images in Compute Engine incur
charges. See [](https://cloud.google.com/compute/docs/images/importing-virtual-disks#resource_cleanup).
Troubleshooting: Image import/export tools rely on CloudBuild default
behavior. They has been using the default CloudBuild service account in
order to import/export images to/from Google Cloud Platform. However,
Cloud Build has changed this default behavior and in new projects a
custom user managed service account may need to be provided to perform
the builds. If you get a CloudBuild service account related error, run
gcloud with --cloudbuild-service-account=<custom service account>.
See `gcloud compute images import --help` for details.
""",
'EXAMPLES': """
To import a centos-7 VMDK file, run:
$ {command} myimage-name --os=centos-7 --source-file=mysourcefile
To import a data disk without operating system, run:
$ {command} myimage-name --data-disk --source-file=mysourcefile
""",
}