On Friday August 14th. from 9 p.m. to 9:30 p.m., we will upgrade GitLab software and its cloud platform release. Service may not be available during this maintenance window. Please schedule your work accordingly.

Commit 0eb0b408 authored by Marcello Golfieri's avatar Marcello Golfieri

parameterized every command for OS specific, only Darwin currently supported

parent 88aa3713
......@@ -32,8 +32,24 @@ class Migrator(object):
self.dest.setup_dest_dir()
def migrate(self):
logger.debug("OS_SPECIFICS => {}".format(self.src.OS_SPECIFICS))
# Usually users have a .backup folder that doesn't play well with gdrive client, and .gdriveignore does not work.
BACKUP_ENABLED = (
True
if os.path.exists(os.path.join(self.src.AFS_USER_HOME + ".backup"))
else False
)
if BACKUP_ENABLED:
# Removing such .backup mountpoint
if os.system(self.src.os_spec("REMOVE_AFS_BACKUP_FOLDER")) != 0:
logger.debug(
"Failed command {}".format(
self.src.os_spec("REMOVE_AFS_BACKUP_FOLDER")
)
)
# Main migration sync
command = (
"gdrive sync upload --keep-local "
"gdrive sync upload --keep-local --delete-extraneous "
+ self.src.AFS_USER_HOME
+ " "
+ self.dest.DEST_DIR_ID
......@@ -48,6 +64,21 @@ class Migrator(object):
self.dest.DEST_DIR_ID,
)
)
if BACKUP_ENABLED:
# Restore such .backup mount once done.
if (
os.system(
self.src.os_spec("RESTORE_AFS_BACKUP_FOLDER").replace(
"SUNET", self.src.SUNET
)
)
!= 0
):
logger.debug(
"Failed command {}".format(
self.src.os_spec("RESTORE_AFS_BACKUP_FOLDER")
)
)
class SuDatastore(object):
......@@ -55,11 +86,7 @@ class SuDatastore(object):
HOME = None
OS = None
OS_SPECIFICS = {
"Darwin": {
"HOME_PATH": "/afs/ir/users/",
"GDRIVE_INSTALL_STEPS": 'Run "brew install gdrive" in your shell',
"DEFAULT_BROWSER_CMD": "open",
}
"Darwin": {"HOME_PATH": "/afs/ir/users/", "DEFAULT_BROWSER_CMD": "open"}
}
def __init__(self, sunet=None):
......@@ -68,8 +95,16 @@ class SuDatastore(object):
self.HOME = os.environ.get("HOME")
def os_spec(self, key):
logger.debug(
"os_spec() is running {} => {}".format(
key, self.OS_SPECIFICS.get(self.OS).get(key)
)
)
return self.OS_SPECIFICS.get(self.OS).get(key)
def os_spec_cmd(self, key):
return self.os_spec(key).split()
def shell_output(self, command, **kwargs):
options = {"stderr": subprocess.STDOUT, "shell": True}
options.update(kwargs)
......@@ -82,19 +117,29 @@ class SuAFS(SuDatastore):
def __init__(self, sunet=None, afs_user_home=None):
super().__init__(sunet)
self.OS_SPECIFICS["Darwin"].update(
{
"AKLOG": "aklog",
"FS_LA": "fs la",
"FS_GETCALLERACCESS": "fs getcalleraccess",
"KLIST_TEST_VALID_TICKET": "klist -t",
"REMOVE_AFS_BACKUP_FOLDER": "fs rm .backup",
"RESTORE_AFS_BACKUP_FOLDER": "fs mk .backup user.SUNET.backup",
}
)
if afs_user_home:
self.AFS_USER_HOME = afs_user_home
else:
self.AFS_USER_HOME = os.path.join(
self.os_spec("HOME_PATH"), self.SUNET[0], self.SUNET[1], self.SUNET
)
if subprocess.call(["klist", "-t"]) != 0:
if subprocess.call(self.os_spec_cmd("KLIST_TEST_VALID_TICKET")) != 0:
raise Exception(
'User {0} has no valid kerberos principal, obtain it with "kinit {0}"'.format(
self.SUNET
)
)
if subprocess.call(["aklog"]) != 0:
if subprocess.call(self.os_spec_cmd("AKLOG")) != 0:
raise Exception('aklog was not successful for user {0}"'.format(self.SUNET))
if not os.path.isdir(self.AFS_USER_HOME):
raise Exception(
......@@ -103,7 +148,7 @@ class SuAFS(SuDatastore):
)
)
os.chdir(self.AFS_USER_HOME)
acl_output = subprocess.check_output(["fs", "la"])
acl_output = self.shell_output(self.os_spec("FS_LA"))
logger.debug("fs la => {}".format(acl_output))
non_system_matches = re.findall(
rb"^\s+(?!.*system:).*$", acl_output, re.MULTILINE
......@@ -117,7 +162,7 @@ class SuAFS(SuDatastore):
if (
not re.search(
rb"^Callers access to \. is rlidwka$",
self.shell_output("fs getcalleraccess"),
self.shell_output(self.os_spec("FS_GETCALLERACCESS")),
)
and self.AFS_HOME_OWNER != self.SUNET
):
......@@ -133,7 +178,17 @@ class SuGDrive(SuDatastore):
def __init__(self, sunet=None, dest_home_path="imported_AFS"):
super().__init__(sunet)
if subprocess.call(["which", "gdrive"]) != 0:
self.OS_SPECIFICS["Darwin"].update(
{
"GDRIVE_INSTALL_STEPS": 'Run "brew install gdrive" in your shell',
"WHICH_GDRIVE": "which gdrive",
"GDRIVE_LIST": "gdrive list",
"GDRIVE_MKDIR": "gdrive mkdir 'DIRECTORY'",
"CHECK_IF_DIRECTORY_EXIST_IN_ROOT": "gdrive list -q \"name = 'DIRECTORY' and 'root' in parents and trashed=false\"",
}
)
if subprocess.call(self.os_spec_cmd("WHICH_GDRIVE")) != 0:
raise Exception(
'Gdrive CLI app not installed, please install it. {0}"'.format(
self.os_spec("GDRIVE_INSTALL_STEPS")
......@@ -141,7 +196,7 @@ class SuGDrive(SuDatastore):
)
if not os.path.exists(os.path.join(self.HOME + ".gdrive/token_v2.json")):
try:
self.shell_output("gdrive list", input=b"\n")
self.shell_output(self.os_spec("GDRIVE_LIST"), input=b"\n")
except subprocess.CalledProcessError as e:
print(
'After you have copied the token from the browser window that just displayed, run "gdrive list" from your command line and paste it when asked'
......@@ -165,12 +220,14 @@ class SuGDrive(SuDatastore):
def setup_dest_dir(self, dirname="AFS_HOME"):
self.DEST_DIR = dirname
listing = self.shell_output(
"gdrive list -q \"name = '{}' and 'root' in parents and trashed=false\"".format(
dirname
self.os_spec("CHECK_IF_DIRECTORY_EXIST_IN_ROOT").replace(
"DIRECTORY", dirname
)
)
if dirname.encode() not in listing:
output = self.shell_output("gdrive mkdir '{}'".format(dirname))
output = self.shell_output(
self.os_spec("GDRIVE_MKDIR").replace("DIRECTORY", dirname)
)
match = re.match(rb"^Directory (.*) created$", output)
if match:
self.DEST_DIR_ID = match.groups()[0].decode("utf-8")
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment