diff --git a/.gitignore b/.gitignore index 63d6663..77f68eb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,8 @@ tusd/data cover.out data/ +.infra/env.sh +.infra/env.infra.sh +.infra/ssh/tusd.pem +.infra/Frey-residu* +.infra/ssh/frey-tusd.pem diff --git a/.infra/Frey-state-terraform.tfstate b/.infra/Frey-state-terraform.tfstate new file mode 100644 index 0000000..5a2828e --- /dev/null +++ b/.infra/Frey-state-terraform.tfstate @@ -0,0 +1,117 @@ +{ + "version": 1, + "serial": 25, + "modules": [ + { + "path": [ + "root" + ], + "outputs": { + "endpoint": "http://master.tus.io:80/", + "public_address": "ec2-54-166-147-2.compute-1.amazonaws.com", + "public_addresses": "ec2-54-166-147-2.compute-1.amazonaws.com" + }, + "resources": { + "aws_instance.tusd": { + "type": "aws_instance", + "primary": { + "id": "i-16ef798d", + "attributes": { + "ami": "ami-9bce7af0", + "availability_zone": "us-east-1d", + "ebs_block_device.#": "0", + "ebs_optimized": "false", + "ephemeral_block_device.#": "0", + "iam_instance_profile": "", + "id": "i-16ef798d", + "instance_state": "running", + "instance_type": "c3.large", + "key_name": "infra-tusd", + "monitoring": "false", + "private_dns": "ip-10-99-128-27.ec2.internal", + "private_ip": "10.99.128.27", + "public_dns": "ec2-54-166-147-2.compute-1.amazonaws.com", + "public_ip": "54.166.147.2", + "root_block_device.#": "1", + "root_block_device.0.delete_on_termination": "true", + "root_block_device.0.iops": "24", + "root_block_device.0.volume_size": "8", + "root_block_device.0.volume_type": "gp2", + "security_groups.#": "1", + "security_groups.1898427277": "fw-tusd-main", + "source_dest_check": "true", + "subnet_id": "", + "tags.#": "1", + "tags.Name": "master.tus.io", + "tenancy": "default", + "vpc_security_group_ids.#": "0" + }, + "meta": { + "schema_version": "1" + } + } + }, + "aws_route53_record.www": { + "type": "aws_route53_record", + "depends_on": [ + "aws_instance.tusd" + ], + "primary": { + "id": "Z3IT8X6U91XY1P_master.tus.io_CNAME", + "attributes": { + "failover": "", + "fqdn": "master.tus.io", + "health_check_id": "", + "id": "Z3IT8X6U91XY1P_master.tus.io_CNAME", + "name": "master.tus.io", + "records.#": "1", + "records.3871998453": "ec2-54-166-147-2.compute-1.amazonaws.com", + "set_identifier": "", + "ttl": "300", + "type": "CNAME", + "weight": "-1", + "zone_id": "Z3IT8X6U91XY1P" + } + } + }, + "aws_security_group.fw-tusd-main": { + "type": "aws_security_group", + "primary": { + "id": "sg-60cc290d", + "attributes": { + "description": "Infra tusd", + "egress.#": "0", + "id": "sg-60cc290d", + "ingress.#": "3", + "ingress.2214680975.cidr_blocks.#": "1", + "ingress.2214680975.cidr_blocks.0": "0.0.0.0/0", + "ingress.2214680975.from_port": "80", + "ingress.2214680975.protocol": "tcp", + "ingress.2214680975.security_groups.#": "0", + "ingress.2214680975.self": "false", + "ingress.2214680975.to_port": "80", + "ingress.2541437006.cidr_blocks.#": "1", + "ingress.2541437006.cidr_blocks.0": "0.0.0.0/0", + "ingress.2541437006.from_port": "22", + "ingress.2541437006.protocol": "tcp", + "ingress.2541437006.security_groups.#": "0", + "ingress.2541437006.self": "false", + "ingress.2541437006.to_port": "22", + "ingress.516175195.cidr_blocks.#": "1", + "ingress.516175195.cidr_blocks.0": "0.0.0.0/0", + "ingress.516175195.from_port": "8080", + "ingress.516175195.protocol": "tcp", + "ingress.516175195.security_groups.#": "0", + "ingress.516175195.self": "false", + "ingress.516175195.to_port": "8080", + "name": "fw-tusd-main", + "owner_id": "402421253186", + "tags.#": "0", + "vpc_id": "" + } + } + } + } + } + ] +} diff --git a/.infra/Frey-state-terraform.tfstate.backup b/.infra/Frey-state-terraform.tfstate.backup new file mode 100644 index 0000000..2904501 --- /dev/null +++ b/.infra/Frey-state-terraform.tfstate.backup @@ -0,0 +1,119 @@ +{ + "version": 1, + "serial": 24, + "modules": [ + { + "path": [ + "root" + ], + "outputs": { + "endpoint": "http://master.tus.io:80/", + "public_address": "ec2-54-166-147-2.compute-1.amazonaws.com", + "public_addresses": "ec2-54-166-147-2.compute-1.amazonaws.com" + }, + "resources": { + "aws_instance.tusd": { + "type": "aws_instance", + "primary": { + "id": "i-16ef798d", + "attributes": { + "ami": "ami-9bce7af0", + "availability_zone": "us-east-1d", + "ebs_block_device.#": "0", + "ebs_optimized": "false", + "ephemeral_block_device.#": "0", + "iam_instance_profile": "", + "id": "i-16ef798d", + "instance_state": "running", + "instance_type": "c3.large", + "key_name": "infra-tusd", + "monitoring": "false", + "private_dns": "ip-10-99-128-27.ec2.internal", + "private_ip": "10.99.128.27", + "public_dns": "ec2-54-166-147-2.compute-1.amazonaws.com", + "public_ip": "54.166.147.2", + "root_block_device.#": "1", + "root_block_device.0.delete_on_termination": "true", + "root_block_device.0.iops": "24", + "root_block_device.0.volume_size": "8", + "root_block_device.0.volume_type": "gp2", + "security_groups.#": "1", + "security_groups.1898427277": "fw-tusd-main", + "source_dest_check": "true", + "subnet_id": "", + "tags.#": "1", + "tags.Name": "master.tus.io", + "tenancy": "default", + "vpc_security_group_ids.#": "0" + }, + "meta": { + "schema_version": "1" + } + } + }, + "aws_route53_record.www": { + "type": "aws_route53_record", + "depends_on": [ + "aws_instance.tusd" + ], + "primary": { + "id": "Z3IT8X6U91XY1P_master.tus.io_CNAME", + "attributes": { + "failover": "", + "fqdn": "master.tus.io", + "health_check_id": "", + "id": "Z3IT8X6U91XY1P_master.tus.io_CNAME", + "name": "master.tus.io", + "records.#": "1", + "records.3871998453": "ec2-54-166-147-2.compute-1.amazonaws.com", + "set_identifier": "", + "ttl": "300", + "type": "CNAME", + "weight": "-1", + "zone_id": "Z3IT8X6U91XY1P" + } + } + }, + "aws_security_group.fw-tusd-main": { + "type": "aws_security_group", + "primary": { + "id": "sg-60cc290d", + "attributes": { + "description": "Infra tusd", + "egress.#": "0", + "id": "sg-60cc290d", + "ingress.#": "3", + "ingress.2214680975.cidr_blocks.#": "1", + "ingress.2214680975.cidr_blocks.0": "0.0.0.0/0", + "ingress.2214680975.from_port": "80", + "ingress.2214680975.protocol": "tcp", + "ingress.2214680975.security_groups.#": "0", + "ingress.2214680975.self": "false", + "ingress.2214680975.to_port": "80", + "ingress.516175195.cidr_blocks.#": "1", + "ingress.516175195.cidr_blocks.0": "0.0.0.0/0", + "ingress.516175195.from_port": "8080", + "ingress.516175195.protocol": "tcp", + "ingress.516175195.security_groups.#": "0", + "ingress.516175195.self": "false", + "ingress.516175195.to_port": "8080", + "ingress.614077637.cidr_blocks.#": "3", + "ingress.614077637.cidr_blocks.0": "62.163.187.106/32", + "ingress.614077637.cidr_blocks.1": "84.146.0.0/16", + "ingress.614077637.cidr_blocks.2": "24.134.75.132/32", + "ingress.614077637.from_port": "22", + "ingress.614077637.protocol": "tcp", + "ingress.614077637.security_groups.#": "0", + "ingress.614077637.self": "false", + "ingress.614077637.to_port": "22", + "name": "fw-tusd-main", + "owner_id": "402421253186", + "tags.#": "0", + "vpc_id": "" + } + } + } + } + } + ] +} diff --git a/.infra/Freyfile.toml b/.infra/Freyfile.toml new file mode 100644 index 0000000..248cca5 --- /dev/null +++ b/.infra/Freyfile.toml @@ -0,0 +1,175 @@ +[global] + appname = "tusd" + approot = "/srv/tusd" + +[global.ssh] + key_dir = "./ssh" + +[global.ansiblecfg.privilege_escalation] + become = true + +[global.ansiblecfg.defaults] + host_key_checking = false + +[infra.provider.aws] + access_key = "${var.FREY_AWS_ACCESS_KEY}" + region = "us-east-1" + secret_key = "${var.FREY_AWS_SECRET_KEY}" + +[infra.variable.ami.default] + us-east-1 = "ami-9bce7af0" +[infra.variable.region] + default = "us-east-1" + +[infra.output.public_address] + value = "${aws_instance.tusd.0.public_dns}" +[infra.output.public_addresses] + value = "${join(\"\n\", aws_instance.tusd.*.public_dns)}" +[infra.output.endpoint] + value = "http://${aws_route53_record.www.name}:80/" + +[infra.resource.aws_instance.tusd] + ami = "${lookup(var.ami, var.region)}" + instance_type = "c3.large" + key_name = "infra-tusd" + security_groups = ["fw-tusd-main"] + [infra.resource.aws_instance.tusd.connection] + key_file = "{{{config.global.ssh.privatekey_file}}}" + user = "{{{config.global.ssh.user}}}" + [infra.resource.aws_instance.tusd.tags] + Name = "${var.FREY_DOMAIN}" + +[infra.resource.aws_route53_record.www] + name = "${var.FREY_DOMAIN}" + records = ["${aws_instance.tusd.public_dns}"] + ttl = "300" + type = "CNAME" + zone_id = "${var.FREY_AWS_ZONE_ID}" + +[infra.resource.aws_security_group.fw-tusd-main] + description = "Infra tusd" + name = "fw-tusd-main" + [[infra.resource.aws_security_group.fw-tusd-main.ingress]] + cidr_blocks = [ "0.0.0.0/0" ] + from_port = 8080 + protocol = "tcp" + to_port = 8080 + [[infra.resource.aws_security_group.fw-tusd-main.ingress]] + cidr_blocks = [ "0.0.0.0/0" ] + from_port = 80 + protocol = "tcp" + to_port = 80 + [[infra.resource.aws_security_group.fw-tusd-main.ingress]] + cidr_blocks = [ "0.0.0.0/0" ] + from_port = 22 + protocol = "tcp" + to_port = 22 + +[[install.playbooks]] + hosts = "tusd" + name = "Install tusd" + + [[install.playbooks.roles]] + role = "{{{init.paths.roles_dir}}}/apt/v1.0.0" + apt_packages = [ + "apg", + "build-essential", + "curl", + "git-core", + "htop", + "iotop", + "libpcre3", + "logtail", + "mlocate", + "mtr", + "psmisc", + "telnet", + "vim", + "wget", + ] + + [[install.playbooks.roles]] + role = "{{{init.paths.roles_dir}}}/unattended-upgrades/v1.2.0" + + [[install.playbooks.tasks]] + lineinfile = "dest=/home/{{{config.global.ssh.user}}}/.bashrc line=\"alias wtf='sudo tail -f /var/log/*{log,err} /var/log/{dmesg,messages,*{,/*}{log,err}}'\" owner={{{config.global.ssh.user}}} group={{{config.global.ssh.user}}} mode=0644 backup=yes" + name = "Common | Add convenience shortcut wtf" + + [[install.playbooks.tasks]] + lineinfile = "dest=/home/{{{config.global.ssh.user}}}/.bashrc line=\"cd {{{config.global.approot}}}/current || true\" owner={{{config.global.ssh.user}}} group={{{config.global.ssh.user}}} mode=0644 backup=yes" + name = "Common | Install login" + + [[install.playbooks.tasks]] + name = "Common | Set motd" + copy = "content='Welcome to {{lookup('env', 'FREY_DOMAIN')}}' dest=/etc/motd owner=root group=root mode=0644 backup=yes" + + [[install.playbooks.tasks]] + name = "Common | Set timezone variables" + copy = "content='Etc/UTC' dest=/etc/timezone owner=root group=root mode=0644 backup=yes" + notify = [ "Common | Update timezone" ] + + [[install.playbooks.handlers]] + name = "Common | Update timezone" + command = "dpkg-reconfigure --frontend noninteractive tzdata" + + +[[setup.playbooks]] + hosts = "tusd" + name = "Setup tusd" + + [[setup.playbooks.roles]] + role = "{{{init.paths.roles_dir}}}/upstart/v1.0.0" + upstart_command = "./tusd -port=8080 -dir=/mnt/tusd-data -store-size=10737418240" + upstart_description = "tusd server" + upstart_name = "{{{config.global.appname}}}" + upstart_pidfile_path = "{{{config.global.approot}}}/shared/{{{config.global.appname}}}.pid" + upstart_respawn = true + upstart_respawn_limit = true + upstart_runtime_root = "{{{config.global.approot}}}/current/tusd_linux_amd64" + upstart_user = "www-data" + + [[setup.playbooks.roles]] + role = "{{{init.paths.roles_dir}}}/rsyslog/v3.0.1" + [setup.playbooks.roles.rsyslog_rsyslog_d_files.49-tusd] + # Send {{{config.global.appname}}} messages to a dedicated logfile + directives = [ "& stop" ] + [[setup.playbooks.roles.rsyslog_rsyslog_d_files.49-tusd.rules]] + rule = ":programname, startswith, \"{{{config.global.appname}}}\"" + logpath = "{{{config.global.approot}}}/shared/logs/{{{config.global.appname}}}.log" + + [[setup.playbooks.roles]] + role = "{{{init.paths.roles_dir}}}/fqdn/v1.0.0" + fqdn = "{{lookup('env', 'FREY_DOMAIN')}}" + + [[setup.playbooks.tasks]] + file = "path=/mnt/tusd-data state=directory owner=www-data group=www-data mode=0755 recurse=yes" + name = "tusd | Create tusd data dir" + +[[deploy.playbooks]] + hosts = "tusd" + name = "Deploy tusd" + + [[deploy.playbooks.roles]] + # @todo On travis, we could directly use the generated + # /home/travis/gopath/src/github.com/tus/tusd/snapshot/tusd_linux_amd64.tar.gz + role = "{{{init.paths.roles_dir}}}/deploy/v1.4.0" + ansistrano_get_url = "https://github.com/tus/tusd/releases/download/0.1.2/tusd_linux_amd64.tar.gz" + ansistrano_deploy_to = "{{{config.global.approot}}}" + ansistrano_deploy_via = "download_unarchive" + ansistrano_group = "ubuntu" + + [[deploy.playbooks.tasks]] + name = "tusd | Set file attributes" + file = "path={{{config.global.approot}}}/current/tusd_linux_amd64/tusd mode=0755 owner=www-data group=www-data" + +[[restart.playbooks]] + hosts = "tusd" + name = "Restart tusd" + + [[restart.playbooks.tasks]] + shell = "iptables -t nat -A PREROUTING -i eth0 -p tcp --dport 80 -j REDIRECT --to-port 8080" + name = "tusd | Redirect HTTP traffic to tusd" + + [[restart.playbooks.tasks]] + action = "service name=tusd state=restarted" + name = "tusd | Restart" diff --git a/.infra/README.md b/.infra/README.md new file mode 100644 index 0000000..eac03a4 --- /dev/null +++ b/.infra/README.md @@ -0,0 +1,3 @@ +This folder is charge of launching master.tus.io via [Frey](https://github.com/kvz/frey). +You could re-use bits of this, but you could also run tusd any other way you want. +It's not a requirement for running a tusd server. diff --git a/.infra/env.example.sh b/.infra/env.example.sh new file mode 100644 index 0000000..0fe3534 --- /dev/null +++ b/.infra/env.example.sh @@ -0,0 +1,10 @@ +# Rename this file to env.sh, it will be kept out of Git. +# So suitable for adding secret keys and such + +# export DEBUG="frey:*" +# export FREY_DOMAIN="master.tus.io" +# export FREY_ENCRYPTION_SECRET="***" + +# source env.sh +# travis encrypt --add env.global "FREY_DOMAIN=${FREY_DOMAIN}" +# travis encrypt --add env.global "FREY_ENCRYPTION_SECRET=${FREY_ENCRYPTION_SECRET}" diff --git a/.infra/env.infra.example.sh b/.infra/env.infra.example.sh new file mode 100644 index 0000000..e961ab3 --- /dev/null +++ b/.infra/env.infra.example.sh @@ -0,0 +1,5 @@ +source env.sh + +# export FREY_AWS_ACCESS_KEY="***" +# export FREY_AWS_SECRET_KEY="***" +# export FREY_AWS_ZONE_ID="***" diff --git a/.infra/ssh/frey-tusd.pem.cast5 b/.infra/ssh/frey-tusd.pem.cast5 new file mode 100644 index 0000000..215171f Binary files /dev/null and b/.infra/ssh/frey-tusd.pem.cast5 differ diff --git a/.infra/ssh/frey-tusd.pub b/.infra/ssh/frey-tusd.pub new file mode 100644 index 0000000..e69de29 diff --git a/.travis.yml b/.travis.yml index 13f0727..c92a4b5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,30 +5,30 @@ go: - 1.5 - 1.6rc2 - tip - -sudo: false - +sudo: required +cache: + apt: true + directories: + - $HOME/.gimme + - $HOME/.frey env: global: - GO15VENDOREXPERIMENT=1 - + - secure: Ls76vWCzitRPJ4gyIAj7SdAUkbp2hhYd0zsDjHXriZA+KmlrZLsr4L3QcOaX+MMjMyEURZMS43VVaygP6Y8oF5yv4wYBHKgYh0ZW96pWrIS++0B5lo+2U+JZtTKAfLnOLHUQfLL5h4fg9+oa4mlcuH+3FvaCwO0TJjLU1b0q3Fc= + - secure: aAfgsISNfOC/8FDZSctCJr/JnxxnPkd1o0Zcqc+mTw6NXUa4T3o+aqbaY/VK5lk7Pru9exs5j+HdaeGhQrLMZJca803Sg1r0m68+h/2bS58p9zjyPk7aULCSAF7EUlqT041QQ5UBJV7gpIxFW1nyD6vL0ZBW1wA1k1PpxTjznPA= matrix: allow_failures: - go: tip - install: -- export PACKAGES=$(find ./ -maxdepth 1 -type d -not \( -name ".git" -or -name "cmd" -or -name "vendor" -or -name "data" -or -name ".hooks" \)) +- export PACKAGES=$(find ./ -maxdepth 1 -type d -not \( -name ".git" -or -name "cmd" -or -name ".infra" -or -name "vendor" -or -name "data" -or -name ".hooks" \)) - rsync -r ./vendor/ $GOPATH/src - script: - go test $PACKAGES - before_deploy: - export GOROOT_BOOTSTRAP=$GOROOT - go get github.com/laher/goxc - goxc -t -bc="linux darwin windows" - goxc -d=./ -wd=./cmd/tusd -bc="linux darwin windows" -build-ldflags="-X main.VersionName=$TRAVIS_TAG -X main.GitCommit=$TRAVIS_COMMIT -X 'main.BuildDate=$(date --utc)'" - deploy: provider: releases api_key: @@ -45,8 +45,10 @@ deploy: - snapshot/tusd_windows_386.zip - snapshot/tusd_windows_amd64.zip skip_cleanup: true - on: tags: true go: 1.5 repo: tus/tusd +after_deploy: +- .infra/scripts/rebuild-env.sh +- if [ "${TRAVIS_PULL_REQUEST}" == "false" ] && [ "${TRAVIS_BRANCH}" == "master" ]; then (make frey && frey setup --force-yes --projectDir .infra); else echo "Skipping deploy for non-master/PRs"; fi diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..0b86608 --- /dev/null +++ b/Makefile @@ -0,0 +1,29 @@ +FREY_VERSION := 0.3.13 + +.PHONY: frey +frey: + @grep $(FREY_VERSION) node_modules/frey/package.json 2>&1 > /dev/null || npm install frey@$(FREY_VERSION) + +.PHONY: provision +provision: frey + @source env.sh && node_modules/.bin/frey install + +.PHONY: deploy +deploy: + @cd .infra && source env.sh && frey setup + +.PHONY: launch +launch: + @cd .infra && source env.infra.sh && frey infra + +.PHONY: console +console: + @cd .infra && source env.sh && frey remote + +.PHONY: deploy-localfrey +deploy-localfrey: + @cd .infra && source env.sh && babel-node ${HOME}/code/frey/src/cli.js setup + +.PHONY: console-localfrey +console-localfrey: + @cd .infra && source env.sh && babel-node ${HOME}/code/frey/src/cli.js remote