File: //snap/google-cloud-cli/current/lib/googlecloudsdk/command_lib/app/deployables.py
# -*- coding: utf-8 -*- #
# Copyright 2017 Google LLC. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Utilities for deriving services and configs from paths.
Paths are typically given as positional params, like
`gcloud app deploy <path1> <path2>...`.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
import collections
import os
from googlecloudsdk.api_lib.app import env
from googlecloudsdk.api_lib.app import yaml_parsing
from googlecloudsdk.command_lib.app import exceptions
from googlecloudsdk.core import log
from googlecloudsdk.core.util import files
_STANDARD_APP_YAML_URL = (
'https://cloud.google.com/appengine/docs/standard/reference/app-yaml')
_FLEXIBLE_APP_YAML_URL = (
'https://cloud.google.com/appengine/docs/flexible/reference/app-yaml')
APP_YAML_INSTRUCTIONS = (
'using the directions at {flex} (App Engine flexible environment) or {std} '
'(App Engine standard environment) under the tab for your language.'
).format(flex=_FLEXIBLE_APP_YAML_URL, std=_STANDARD_APP_YAML_URL)
FINGERPRINTING_WARNING = (
'As an alternative, create an app.yaml file yourself ' +
APP_YAML_INSTRUCTIONS)
NO_YAML_ERROR = (
'An app.yaml (or appengine-web.xml) file is required to deploy this '
'directory as an App Engine application. Create an app.yaml file '
+ APP_YAML_INSTRUCTIONS)
class Service(object):
"""Represents data around a deployable service.
Attributes:
descriptor: str, File path to the original deployment descriptor, which is
either a `<service>.yaml` or an `appengine-web.xml`.
source: str, Path to the original deployable artifact or directory, which
is typically the original source directory, but could also be an artifact
such as a fat JAR file.
service_info: yaml_parsing.ServiceYamlInfo, Info parsed from the
`<service>.yaml` file. Note that service_info.file may point to a
file in a staged directory.
upload_dir: str, Path to the source directory. If staging is required, this
points to the staged directory.
service_id: str, the service id.
path: str, File path to the staged deployment `<service>.yaml` descriptor
or to the original one, if no staging is used.
"""
def __init__(self, descriptor, source, service_info, upload_dir):
self.descriptor = descriptor
self.source = source
self.service_info = service_info
self.upload_dir = upload_dir
@property
def service_id(self):
return self.service_info.module
@property
def path(self):
return self.service_info.file
@classmethod
def FromPath(cls, path, stager, path_matchers, appyaml):
"""Return a Service from a path using staging if necessary.
Args:
path: str, Unsanitized absolute path, may point to a directory or a file
of any type. There is no guarantee that it exists.
stager: staging.Stager, stager that will be invoked if there is a runtime
and environment match.
path_matchers: List[Function], ordered list of functions on the form
fn(path, stager), where fn returns a Service or None if no match.
appyaml: str or None, the app.yaml location to used for deployment.
Returns:
Service, if one can be derived, else None.
"""
for matcher in path_matchers:
service = matcher(path, stager, appyaml)
if service:
return service
return None
def ServiceYamlMatcher(path, stager, appyaml):
"""Generate a Service from an <service>.yaml source path.
This function is a path matcher that returns if and only if:
- `path` points to either a `<service>.yaml` or `<app-dir>` where
`<app-dir>/app.yaml` exists.
- the yaml-file is a valid <service>.yaml file.
If the runtime and environment match an entry in the stager, the service will
be staged into a directory.
Args:
path: str, Unsanitized absolute path, may point to a directory or a file of
any type. There is no guarantee that it exists.
stager: staging.Stager, stager that will be invoked if there is a runtime
and environment match.
appyaml: str or None, the app.yaml location to used for deployment.
Raises:
staging.StagingCommandFailedError, staging command failed.
Returns:
Service, fully populated with entries that respect a potentially
staged deployable service, or None if the path does not match the
pattern described.
"""
descriptor = path if os.path.isfile(path) else os.path.join(path,
'app.yaml')
_, ext = os.path.splitext(descriptor)
if os.path.exists(descriptor) and ext in ['.yaml', '.yml']:
app_dir = os.path.dirname(descriptor)
service_info = yaml_parsing.ServiceYamlInfo.FromFile(descriptor)
staging_dir = stager.Stage(descriptor, app_dir, service_info.runtime,
service_info.env, appyaml)
# If staging, stage, get stage_dir
return Service(descriptor, app_dir, service_info, staging_dir or app_dir)
return None
def JarMatcher(jar_path, stager, appyaml):
"""Generate a Service from a Java fatjar path.
This function is a path matcher that returns if and only if:
- `jar_path` points to a jar file .
The service will be staged according to the stager as a jar runtime,
which is defined in staging.py.
Args:
jar_path: str, Unsanitized absolute path pointing to a file of jar type.
stager: staging.Stager, stager that will be invoked if there is a runtime
and environment match.
appyaml: str or None, the app.yaml location to used for deployment.
Raises:
staging.StagingCommandFailedError, staging command failed.
Returns:
Service, fully populated with entries that respect a staged deployable
service, or None if the path does not match the pattern described.
"""
_, ext = os.path.splitext(jar_path)
if os.path.exists(jar_path) and ext in ['.jar']:
app_dir = os.path.abspath(os.path.join(jar_path, os.pardir))
descriptor = jar_path
staging_dir = stager.Stage(descriptor, app_dir, 'java-jar', env.STANDARD,
appyaml)
yaml_path = os.path.join(staging_dir, 'app.yaml')
service_info = yaml_parsing.ServiceYamlInfo.FromFile(yaml_path)
return Service(descriptor, app_dir, service_info, staging_dir)
return None
def PomXmlMatcher(path, stager, appyaml):
"""Generate a Service from an Maven project source path.
This function is a path matcher that returns true if and only if:
- `path` points to either a Maven `pom.xml` or `<maven=project-dir>` where
`<maven-project-dir>/pom.xml` exists.
If the runtime and environment match an entry in the stager, the service will
be staged into a directory.
Args:
path: str, Unsanitized absolute path, may point to a directory or a file of
any type. There is no guarantee that it exists.
stager: staging.Stager, stager that will be invoked if there is a runtime
and environment match.
appyaml: str or None, the app.yaml location to used for deployment.
Raises:
staging.StagingCommandFailedError, staging command failed.
Returns:
Service, fully populated with entries that respect a potentially
staged deployable service, or None if the path does not match the
pattern described.
"""
descriptor = path if os.path.isfile(path) else os.path.join(path, 'pom.xml')
filename = os.path.basename(descriptor)
if os.path.exists(descriptor) and filename == 'pom.xml':
app_dir = os.path.dirname(descriptor)
staging_dir = stager.Stage(descriptor, app_dir, 'java-maven-project',
env.STANDARD, appyaml)
yaml_path = os.path.join(staging_dir, 'app.yaml')
service_info = yaml_parsing.ServiceYamlInfo.FromFile(yaml_path)
return Service(descriptor, app_dir, service_info, staging_dir)
return None
def BuildGradleMatcher(path, stager, appyaml):
"""Generate a Service from an Gradle project source path.
This function is a path matcher that returns true if and only if:
- `path` points to either a Gradle `build.gradle` or `<gradle-project-dir>`
where `<gradle-project-dir>/build.gradle` exists.
If the runtime and environment match an entry in the stager, the service will
be staged into a directory.
Args:
path: str, Unsanitized absolute path, may point to a directory or a file of
any type. There is no guarantee that it exists.
stager: staging.Stager, stager that will be invoked if there is a runtime
and environment match.
appyaml: str or None, the app.yaml location to used for deployment.
Raises:
staging.StagingCommandFailedError, staging command failed.
Returns:
Service, fully populated with entries that respect a potentially
staged deployable service, or None if the path does not match the
pattern described.
"""
descriptor = path if os.path.isfile(path) else os.path.join(
path, 'build.gradle')
filename = os.path.basename(descriptor)
if os.path.exists(descriptor) and filename == 'build.gradle':
app_dir = os.path.dirname(descriptor)
staging_dir = stager.Stage(descriptor, app_dir, 'java-gradle-project',
env.STANDARD, appyaml)
yaml_path = os.path.join(staging_dir, 'app.yaml')
service_info = yaml_parsing.ServiceYamlInfo.FromFile(yaml_path)
return Service(descriptor, app_dir, service_info, staging_dir)
return None
def AppengineWebMatcher(path, stager, appyaml):
"""Generate a Service from an appengine-web.xml source path.
This function is a path matcher that returns if and only if:
- `path` points to either `.../WEB-INF/appengine-web.xml` or `<app-dir>` where
`<app-dir>/WEB-INF/appengine-web.xml` exists.
- the xml-file is a valid appengine-web.xml file according to the Java stager.
The service will be staged according to the stager as a java-xml runtime,
which is defined in staging.py.
Args:
path: str, Unsanitized absolute path, may point to a directory or a file of
any type. There is no guarantee that it exists.
stager: staging.Stager, stager that will be invoked if there is a runtime
and environment match.
appyaml: str or None, the app.yaml location to used for deployment.
Raises:
staging.StagingCommandFailedError, staging command failed.
Returns:
Service, fully populated with entries that respect a staged deployable
service, or None if the path does not match the pattern described.
"""
suffix = os.path.join(os.sep, 'WEB-INF', 'appengine-web.xml')
app_dir = path[:-len(suffix)] if path.endswith(suffix) else path
descriptor = os.path.join(app_dir, 'WEB-INF', 'appengine-web.xml')
if not os.path.isfile(descriptor):
return None
xml_file = files.ReadFileContents(descriptor)
if '<application>' in xml_file or '<version>' in xml_file:
log.warning('<application> and <version> elements in ' +
'`appengine-web.xml` are not respected')
staging_dir = stager.Stage(descriptor, app_dir, 'java-xml', env.STANDARD,
appyaml)
if not staging_dir:
# After GA launch of appengine-web.xml support, this should never occur.
return None
yaml_path = os.path.join(staging_dir, 'app.yaml')
service_info = yaml_parsing.ServiceYamlInfo.FromFile(yaml_path)
return Service(descriptor, app_dir, service_info, staging_dir)
def ExplicitAppYamlMatcher(path, stager, appyaml):
"""Use optional app.yaml with a directory or a file the user wants to deploy.
Args:
path: str, Unsanitized absolute path, may point to a directory or a file of
any type. There is no guarantee that it exists.
stager: staging.Stager, stager that will not be invoked.
appyaml: str or None, the app.yaml location to used for deployment.
Returns:
Service, fully populated with entries that respect a staged deployable
service, or None if there is no optional --appyaml flag usage.
"""
if appyaml:
service_info = yaml_parsing.ServiceYamlInfo.FromFile(appyaml)
staging_dir = stager.Stage(appyaml, path, 'generic-copy', service_info.env,
appyaml)
return Service(appyaml, path, service_info, staging_dir)
return None
def UnidentifiedDirMatcher(path, stager, appyaml):
"""Points out to the user that they need an app.yaml to deploy.
Args:
path: str, Unsanitized absolute path, may point to a directory or a file of
any type. There is no guarantee that it exists.
stager: staging.Stager, stager that will not be invoked.
appyaml: str or None, the app.yaml location to used for deployment.
Returns:
None
"""
del stager, appyaml
if os.path.isdir(path):
log.error(NO_YAML_ERROR)
return None
def GetPathMatchers():
"""Get list of path matchers ordered by descending precedence.
Returns:
List[Function], ordered list of functions on the form fn(path, stager),
where fn returns a Service or None if no match.
"""
return [
ServiceYamlMatcher, AppengineWebMatcher, JarMatcher, PomXmlMatcher,
BuildGradleMatcher, ExplicitAppYamlMatcher, UnidentifiedDirMatcher
]
class Services(object):
"""Collection of deployable services."""
def __init__(self, services=None):
"""Instantiate a set of deployable services.
Args:
services: List[Service], optional list of services for quick
initialization.
Raises:
DuplicateServiceError: Two or more services have the same service id.
"""
self._services = collections.OrderedDict()
if services:
for d in services:
self.Add(d)
def Add(self, service):
"""Add a deployable service to the set.
Args:
service: Service, to add.
Raises:
DuplicateServiceError: Two or more services have the same service id.
"""
existing = self._services.get(service.service_id)
if existing:
raise exceptions.DuplicateServiceError(existing.path, service.path,
service.service_id)
self._services[service.service_id] = service
def GetAll(self):
"""Retrieve the service info objects in the order they were added.
Returns:
List[Service], list of services.
"""
return list(self._services.values())
class Configs(object):
"""Collection of config files."""
def __init__(self):
self._configs = collections.OrderedDict()
def Add(self, config):
"""Add a ConfigYamlInfo to the set of configs.
Args:
config: ConfigYamlInfo, the config to add.
Raises:
exceptions.DuplicateConfigError, the config type is already in the set.
"""
config_type = config.config
existing = self._configs.get(config_type)
if existing:
raise exceptions.DuplicateConfigError(existing.file, config.file,
config_type)
self._configs[config_type] = config
def GetAll(self):
"""Retreive the config file objects in the order they were added.
Returns:
List[ConfigYamlInfo], list of config file objects.
"""
return list(self._configs.values())
def GetDeployables(args, stager, path_matchers, appyaml=None):
"""Given a list of args, infer the deployable services and configs.
Given a deploy command, e.g. `gcloud app deploy ./dir other/service.yaml
cron.yaml WEB-INF/appengine-web.xml`, the deployables can be on multiple
forms. This method pre-processes and infers yaml descriptors from the
various formats accepted. The rules are as following:
This function is a context manager, and should be used in conjunction with
the `with` keyword.
1. If `args` is an empty list, add the current directory to it.
2. For each arg:
- If arg refers to a config file, add it to the configs set.
- Else match the arg against the path matchers. The first match will win.
The match will be added to the services set. Matchers may run staging.
Args:
args: List[str], positional args as given on the command-line.
stager: staging.Stager, stager that will be invoked on sources that have
entries in the stager's registry.
path_matchers: List[Function], list of functions on the form
fn(path, stager) ordered by descending precedence, where fn returns
a Service or None if no match.
appyaml: str or None, the app.yaml location to used for deployment.
Raises:
FileNotFoundError: One or more argument does not point to an existing file
or directory.
UnknownSourceError: Could not infer a config or service from an arg.
DuplicateConfigError: Two or more config files have the same type.
DuplicateServiceError: Two or more services have the same service id.
Returns:
Tuple[List[Service], List[ConfigYamlInfo]], lists of deployable services
and configs.
"""
if not args:
args = ['.']
paths = [os.path.abspath(arg) for arg in args]
configs = Configs()
services = Services()
if appyaml:
if len(paths) > 1:
raise exceptions.MultiDeployError()
if not os.path.exists(os.path.abspath(appyaml)):
raise exceptions.FileNotFoundError('File {0} referenced by --appyaml '
'does not exist.'.format(appyaml))
if not os.path.exists(paths[0]):
raise exceptions.FileNotFoundError(paths[0])
for path in paths:
if not os.path.exists(path):
raise exceptions.FileNotFoundError(path)
config = yaml_parsing.ConfigYamlInfo.FromFile(path)
if config:
configs.Add(config)
continue
service = Service.FromPath(path, stager, path_matchers, appyaml)
if service:
services.Add(service)
continue
raise exceptions.UnknownSourceError(path)
return services.GetAll(), configs.GetAll()