Commit 25a78a83 authored by Xu Wang's avatar Xu Wang

docker example: sync git repo in container

parents
.DS_Store
.vagrant
service-account.json
data
\ No newline at end of file
The MIT License (MIT)
Copyright (c) 2017 Xu Wang
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
include envs
export
help: ## this info
@# adapted from https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
@echo '_________________'
@echo '| Make targets: |'
@echo '-----------------'
@cat Makefile | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
build: ## build gcsfuse image
docker-compose build producer
up: ## bring system up, see docker-compose.yml
docker-compose up -d --force-recreate --remove-orphans
down: ## bring system dev-down, see docker-compose.yml
docker-compose down --remove-orphans --volumes
restart: ## restart test system
docker-compose restart
follow: ## docker-compose logs -f
docker-compose logs -f
prune: ## prune local docker containers and images
docker container prune -f
docker image prune -f
docker volume prune -f
gen-key: ## generate git deploy key
@mkdir -p keys
ssh-keygen -t rsa -C "docker-examples" -b 4096 -P "" -f keys/docker-examples
.PHONY: build prune push pull help up down restart follow gen-key
\ No newline at end of file
# Docker Example - Periodically sync data from a git repo
## Prerequisites
#### Set the sync parameters
Edit env file, and set:
```
# sync git repo
REPO=<content git repo git+ssh url>
# git repo deploy key
DEPLOY_KEY=<git repo deploy key>
# pull interval in seconds
INTERVAL=3
```
## Build the alpine-git docker image
```
$ make build
```
## See it works
```
$ make up follow
...
producer_1 | Initialized empty Git repository in /data/.git/
producer_1 | Producer: update /data from git@gitlab.med.stanford.edu:xuwang/docker-examples.git
consumer_1 | Consumer: commit 87e2810bf5dcbfc3f3a5800e9d7c08ec9f97c153 Author: Xu Wang...
...
```
## Shut it down
```
$ make down
```
FROM alpine:edge
RUN apk update && apk upgrade && apk add --no-cache git openssh bash
#!/bin/sh
# just in case ...
chown -R ${UID}:${UID} ${DATA_DIR}
# run with given user
exec sudo -u "#${UID}" gcsfuse \
--foreground \
--key-file=${GOOGLE_APPLICATION_CREDENTIALS} ${BUCKET_NAME} ${DATA_DIR}
\ No newline at end of file
version: '3'
services:
producer:
image: alpine-git
build:
context: ./alpine-git
command: /scripts/producer.sh
env_file:
- ./envs
volumes:
- ./scripts:/scripts
- ./keys:/keys
- data:/data
consumer:
image: alpine
command: /scripts/consumer.sh
env_file:
- ./envs
volumes:
- ./scripts:/scripts
- data:/data
depends_on:
- producer
volumes:
data:
\ No newline at end of file
# git repo deploy key
DEPLOY_KEY=/keys/docker-examples
# git repo to be pulled
REPO=git@gitlab.med.stanford.edu:xuwang/docker-examples.git
# data path
DATA=/data
# pull interval in seconds
INTERVAL=3
-----BEGIN RSA PRIVATE KEY-----
MIIJKQIBAAKCAgEA0RitXEeFWWE92T+0hWJlaQUM2mZ6gsKm/fIE1RdP0a87N8+K
f3tBQncs5Nmho4aLn56F1PaobPa5iq8IozJh3Ue964MlHjQ88KO/eCu5qqtHLSRG
MM8cfI9Z68RmD6Thny7WNn6pmX0N5otmmE/mO8kmmPuGXbakPqCHqvBufFB+jZAJ
9lt0s3IAfzzwf7vWRqURwrwmQxPwVb98/1i9soav+U+aZceoEE8m11GYtaEs2GbK
daFBcrlxgDWPb8rk0EnYgl7yzDvNHQdXEOZCB7CU3FoN6zMcajl1eNwxyUA7fxZx
lfH1j9AxI0DSwIFZ42VcthirHlcAzZSPjkNP4f+FKGtSZa4pkG61eRySC8Cq53uF
nqiDL2W8FEcn9iNIMQrhq3FonBbIDjJ0uC2KzhusEqe5e9AhLHV48BOW/81R3MnE
Xkw6vTvFg36gv52EqUd2npv8zVxJLfas7lutiXI1WKN2CAmzibKxKuCfb8bPRHWg
65nqyiTLYngOUkfb2OUAEGRWuVD53gjtWyD3FR0f84DxpTPYMM8zkeUiDmjUFN+k
S7609JQ/1x9XMYbAqftkdPESydnyKoLMX0md/O/OMrsxUeYE+UgwynkuWDbutFd+
AkALeRI+7Fyb+8JhkTN+5Vp8hAdCE3I/LPDMqXUmfPyfV5Kqg5UKz290m7cCAwEA
AQKCAgA0s2mOZqBfhLjX9syDCefRbmps6YNbTcBVJRi+tJ9zlBzy1iZsNx6dp/fy
vj+VPaYXVxcYNAsDW7QsFi2xJFOyOBDQOT+zuyrY5AywNyUWtRC9efUO/ESLyH27
hMbMjpwnSODI32fTvccxIlV7cjfvPaWyzUJygEQpXMCU/wB56Lfc6j74jth9GfCy
ISPkre015Qjy8gUaxv2+PwM8QJufUX8s8TlTCd18Ltb1UiWRMcxhxpakyz35sm+/
BsDeRl8heFtPkKrVewUZme4B3mcqFQGQAEqdy0YQJ+dJzQXnWX8lavy59EAWPT0F
S6NSzcdgHv4dD16hyP/00MrV6nVJZgxxwrl4Ql1D3DfVWZeJGWAULXo1YOTw/5Sj
p5X4/Vl7uDeM14t3tILAu/rPXqTPXzMKK0gGRJ7gkOV1AiAwgB3OV2WeIuEpwoqI
0rQUHx0GYUXzyrVzGziIX7Je27WZgxq5irkttvbvzQVxYoS3GIYlpoFutT7SFcRl
kH0nyN2vFSFEvleQgLRDFyR/fhCnnO+DUt/rxiTD3zrT2NpNm/+oMIfnm4j8Oamu
Ll165l4ilLcUhpmLENzJQNy/OuwO2Nn0t0NHeIAHU7fRbRkqTPMkNvX9ZvZ070/k
3q+LllTHcdceYDzy7rY4tp7W5jdzEZs9bSsXZoypCdJc1YBAEQKCAQEA9cepdMah
CcxgAT4RDOSR17Ot9SsNK3op2T5roJI1dZ+NfA52BntbdLKtNtucGoRp4dkIpxdp
zFH1Zt9WHsZ3IFgK1ERYZEmmYKJqEGzts4/YXh8ryYwCth9sbVhew9vOoJuQw23E
HdKGHGcNzO9R7P5D4FALDqP8wMi+iAargm4tUSnRGy/NT8M6mr4ANAMs0mCNmuY+
zwhMLCTadE4F6LhHtasrUuef2DVwBI+H9urWzy89kCkSdVZKcHzs1IM8/g4GJMAh
kvSLLafO1h5M6zVjucVJ1S9fhEy6+OSZ+ZNaVK+LBI/DIg5wwZdQwiU7BLb4oHfR
6uYQyTOw3HTzEwKCAQEA2cqEdls+TktM+768cU2IB8IKqj3iH8r35vmoFtwq8d/r
iVGobb4eUgMyEuItsFRRAgY5XDGXUh6r8sPKAeBk9oaZXyrx59Qg7zfgeIKhmlbt
hEHD9emOiVeLSf4yMwrmO+JNcpKQmsBFFVMog+TJHkS1n6bnkdrp9ddhMD5TNE+i
qzvbDJSDK6LB4ND99Vm8YFWyLMpMG5cLmX/uF64WGmulPNFLRFWYY8uLVrt9+6/e
2F4cyd2x5nyJOY1IngfoUZRSIvMFjYAVBg9F4+3dQ6R3m6wxMitcA7hJ5hTFKTrc
Sfyyh9bx4AHlIiv5fRHaegI9B93td9iKYIotDTtlTQKCAQEA2eDKrtOaVKZRgSh1
UvmY9B+7mXg1r5IXYRc9r9WNAJ2VfiEPrcVTGMuAU/4wjy1M88JDMmTGDDOnmS9p
F9UWK50A1JVVymmjzUyNO8xkl+Zvh1iYp42vpaWLqA9LytKjhKsDlMtMrKdA7hLJ
ICATf0UUBNjQRIVI/dm3Vbyb8/8ZuzaXwUMJ/ahy3lSKt5EM/d5MCcIMcdnINOXi
pnG4OUqLf1ejC1Z7093jVJesj041aOtW5uISE+SQDtIf3HAO9KSX+6I64RxZ+Pom
mJe51lS4+CkVv5/ES/L9WMDGYoGm9SW81ilfcWRdvrAWDlWU0SQmmZI+i76xqLtj
ELu38wKCAQBnzzYQTDGIlC/fluyvfOsn93+p4dFUoiRDviA2tT7rB3Z3f6qulH5T
TR6jDZgFgksSkfDpVuHvzJrdHiG9fz3gnAui5w9jFyOToqimCG4K6QBRmB9Vqbu9
DzNfU4CN/9X6GUYdJGs0uAy1u2uGYpjSiqGftaiPRy9ZdQ3oNuzl9EdvIxqoBmn3
dJ7BEBzcw6ccq/w/iTV5uSdYRTgxpVL3yPN7567epYX7m1uRPdExFIU90K4cMkXH
vPiTrBo+6Tn9GJuF+Rxec/yRZNpFaG2c/lnco9jXNv+m7kKbHq9TlbDGu2JFiVjF
E2IqRZen28itez+n7kpwY5Chr3sF4ZfJAoIBAQDKINdOKVj+RQiLYjRTuhFO6v7N
gKq5N3hbTeJQYDggmfgSjjjRNkuqRv5bO07hRAW5XmmNQ9oQAy8vM3aUyGxO+sL2
C2nz3SLjtEwSCUb/epjV9rkC2I71N4LwrDjy7AVgqj502MuXcHyeyHQu3MMX2kra
8X0mZ+BUoi0ffPQbpVcXNCBnbjQhym31DNPaDQpur1tdklAradUP4KNWD7/Ru360
Wd3HAN1e4EUJa+TGif33VvGZ0bXCqMSiGNiN0Z7NKRa2Z1yZK+OJB8GoKrV9WZrv
zbHnp392M3OJnwdO4hfr3CsXR7h5j3kJ57lYkn0mXCPYqYLN0LtmZn+DbXfw
-----END RSA PRIVATE KEY-----
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDRGK1cR4VZYT3ZP7SFYmVpBQzaZnqCwqb98gTVF0/Rrzs3z4p/e0FCdyzk2aGjhoufnoXU9qhs9rmKrwijMmHdR73rgyUeNDzwo794K7mqq0ctJEYwzxx8j1nrxGYPpOGfLtY2fqmZfQ3mi2aYT+Y7ySaY+4ZdtqQ+oIeq8G58UH6NkAn2W3SzcgB/PPB/u9ZGpRHCvCZDE/BVv3z/WL2yhq/5T5plx6gQTybXUZi1oSzYZsp1oUFyuXGANY9vyuTQSdiCXvLMO80dB1cQ5kIHsJTcWg3rMxxqOXV43DHJQDt/FnGV8fWP0DEjQNLAgVnjZVy2GKseVwDNlI+OQ0/h/4Uoa1JlrimQbrV5HJILwKrne4WeqIMvZbwURyf2I0gxCuGrcWicFsgOMnS4LYrOG6wSp7l70CEsdXjwE5b/zVHcycReTDq9O8WDfqC/nYSpR3aem/zNXEkt9qzuW62JcjVYo3YICbOJsrEq4J9vxs9EdaDrmerKJMtieA5SR9vY5QAQZFa5UPneCO1bIPcVHR/zgPGlM9gwzzOR5SIOaNQU36RLvrT0lD/XH1cxhsCp+2R08RLJ2fIqgsxfSZ38784yuzFR5gT5SDDKeS5YNu60V34CQAt5Ej7sXJv7wmGRM37lWnyEB0ITcj8s8MypdSZ8/J9XkqqDlQrPb3Sbtw== docker-examples
\ No newline at end of file
#!/bin/sh
INTERVAL=${INTERVAL:-60}
while true
do
echo Consumer: $(cat /data/git-log.txt )
sleep ${INTERVAL}
done
\ No newline at end of file
#!/bin/sh
# content git repo
REPO=${REPO:-"git@gitlab.med.stanford.edu:xuwang/docker-examples.git"}
# destination dir
DATA=${DATA:-/data}
# git repo deploy key
DEPLOY_KEY=${DEPLOY_KEY:-/keys/docker-examples}
# pull interval in seconds
INTERVAL=${INTERVAL:-60}
git config --global user.name ${GIT_USER:-Docker Examples}
git config --global user.email ${GIT_USER:-docker-examples@exmaple.com}
# git ssh cmd
# why options, see https://www.joedog.org/2012/07/13/ssh-disable-known_hosts-prompt
export GIT_SSH_COMMAND="ssh -q -i $DEPLOY_KEY \
-o 'StrictHostKeyChecking=no' -o 'UserKnownHostsFile=/dev/null'"
# Sync content from git repo to /content on given time interval
if [ ! -d "${DATA}" ]; then mkdir -p ${DATA}; fi
cd $DATA
while true
do
if [ ! -d ".git" ]
then
git init
git remote add origin ${REPO}
fi
echo Producer: update $DATA from ${REPO}
git fetch origin master --depth=1 -f -q
git reset --hard origin/master -q
git log | head > git-log.txt
sleep ${INTERVAL}
done
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment