Skip to content

Commit abcda3a

Browse files
shivnagarajanshivnagarajan
authored
Merge pull request #328 from Shopify/mysql8_arm64_branch
add bits for running arm64 mysql8
2 parents ce94688 + 85120de commit abcda3a

36 files changed

+693
-217
lines changed

‎.github/workflows/start-mysql.sh‎

Lines changed: 29 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,35 +1,49 @@
11
#!/bin/bash
22
set -xe
33

4-
DOCKER_COMPOSE_VERSION=1.29.2
4+
DOCKER_COMPOSE_VERSION=v2.2.3
55

66
sudo apt-get update
77
sudo apt-get install -y netcat-openbsd make gcc
88

99
sudo curl -o /usr/local/bin/docker-compose -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m`
1010
sudo chmod +x /usr/local/bin/docker-compose
1111

12-
docker-compose up -d mysql-1 mysql-2
12+
if [ "$MYSQL_VERSION" == "8.0" ]; then
13+
docker-compose -f docker-compose_8.0.yml up -d mysql-1 mysql-2
14+
else
15+
docker-compose up -d mysql-1 mysql-2
16+
fi
1317

14-
# We need a way to check if the mysql servers have booted or not before running
15-
# the tests and this way is slightly faster than installing mysql-client
18+
MAX_ATTEMPTS=60
1619

17-
wait_for_mysql() {
18-
port=$1
19-
echo "Waiting for MySQL at port $port..."
20+
function wait_for_version () {
2021
attempts=0
21-
while ! nc -w 1 localhost $port | grep -q "mysql"; do
22+
until docker exec -t $1 mysql -N -s -u root -e "select @@version"; do
2223
sleep 1
2324
attempts=$((attempts + 1))
24-
if (( attempts > 60 )); then
25-
echo "ERROR: mysql $port was not started." >&2
26-
exit 1
25+
if (( attempts > $MAX_ATTEMPTS )); then
26+
echo "ERROR: $1 was not started." >&2
27+
exit 1
2728
fi
2829
done
29-
echo "MySQL at port $port has started!"
3030
}
3131

32-
wait_for_mysql 29291
33-
wait_for_mysql 29292
32+
wait_for_configuration () {
33+
attempts=0
34+
# we do need to see the "root@%" user configured, so wait for that
35+
until mysql --port $1 --protocol tcp --skip-password -N -s -u root -e "select host from mysql.user where user = 'root';" 2>/dev/null | grep -q '%'; do
36+
sleep 1
37+
attempts=$((attempts + 1))
38+
if (( attempts > $MAX_ATTEMPTS )); then
39+
echo "ERROR: $1 was not started." >&2
40+
exit 1
41+
fi
42+
done
43+
}
44+
45+
wait_for_version "ghostferry-mysql-1-1"
46+
wait_for_version "ghostferry-mysql-2-1"
3447

35-
docker-compose exec -T mysql-1 mysql -u root -e "select @@version"
48+
wait_for_configuration 29291
49+
wait_for_configuration 29292

‎.github/workflows/tests.yml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Ghostferry tests
1+
name: Ghostferry MySQL 8.0 tests
22

33
on:
44
push:
@@ -12,6 +12,7 @@ jobs:
1212
timeout-minutes: 15
1313
env:
1414
CI: "true"
15+
MYSQL_VERSION: "8.0"
1516
steps:
1617
- uses: actions/checkout@v2
1718

@@ -30,6 +31,7 @@ jobs:
3031
timeout-minutes: 15
3132
env:
3233
CI: "true"
34+
MYSQL_VERSION: "8.0"
3335
steps:
3436
- uses: actions/checkout@v2
3537

@@ -49,6 +51,7 @@ jobs:
4951
env:
5052
CI: "true"
5153
BUNDLE_WITHOUT: "development"
54+
MYSQL_VERSION: "8.0"
5255
steps:
5356
- uses: actions/checkout@v2
5457

‎.github/workflows/tests_5.7.yml‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
name: Ghostferry MySQL 5.7 tests
2+
3+
on:
4+
push:
5+
branches:
6+
- master
7+
pull_request:
8+
9+
jobs:
10+
gh-285:
11+
runs-on: ubuntu-latest
12+
timeout-minutes: 15
13+
env:
14+
CI: "true"
15+
MYSQL_VERSION: "5.7"
16+
steps:
17+
- uses: actions/checkout@v2
18+
19+
- name: Setup Go
20+
uses: actions/setup-go@v2
21+
with:
22+
go-version: 1.16
23+
24+
- name: Starting up MySQL
25+
run: .github/workflows/start-mysql.sh
26+
27+
- name: Running GH-285 test
28+
run: ./examples/gh-285/bugreport.sh
29+
go-test:
30+
runs-on: ubuntu-latest
31+
timeout-minutes: 15
32+
env:
33+
CI: "true"
34+
MYSQL_VERSION: "5.7"
35+
steps:
36+
- uses: actions/checkout@v2
37+
38+
- name: Setup Go
39+
uses: actions/setup-go@v2
40+
with:
41+
go-version: 1.16
42+
43+
- name: Starting up MySQL
44+
run: .github/workflows/start-mysql.sh
45+
46+
- name: Running Golang tests
47+
run: make test-go
48+
ruby-test:
49+
runs-on: ubuntu-latest
50+
timeout-minutes: 15
51+
env:
52+
CI: "true"
53+
MYSQL_VERSION: "5.7"
54+
BUNDLE_WITHOUT: "development"
55+
steps:
56+
- uses: actions/checkout@v2
57+
58+
- name: Setup Golang
59+
uses: actions/setup-go@v2
60+
with:
61+
go-version: 1.16
62+
63+
- name: Setup Ruby
64+
uses: ruby/setup-ruby@v1
65+
with:
66+
ruby-version: 2.7
67+
bundler-cache: true
68+
69+
- name: Starting up MySQL
70+
run: .github/workflows/start-mysql.sh
71+
72+
- name: Running Ruby tests
73+
run: bundle exec ruby test/main.rb
74+

‎dev.yml‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,14 @@
11
name: ghostferry
22

3+
env:
4+
MYSQL_VERSION: "8.0"
5+
36
up:
47
- homebrew:
5-
- mysql
8+
- [email protected]:
9+
or: [[email protected]]
10+
conflicts: [mysql-connector-c, mysql, mysql-client]
11+
612
- ruby: "2.7.3"
713
- bundler
814
- go:
@@ -20,11 +26,17 @@ up:
2026
meet: echo 'go mod failed to download dependencies'; false
2127
- custom:
2228
name: MySQL
23-
met?: docker-compose up -d mysql-1 mysql-2
29+
met?: docker-compose -f docker-compose_8.0.yml up -d mysql-1 mysql-2
2430
meet: echo 'mysql failed to start'; false
25-
down: docker-compose stop mysql-1 mysql-2
31+
down: docker-compose -f docker-compose_8.0.yml stop mysql-1 mysql-2
2632

2733
commands:
2834
test:
29-
desc: Run the test suite.
35+
desc: Run all the tests.
3036
run: make test
37+
test-go:
38+
desc: Run the golang test suite.
39+
run: make test-go
40+
test-ruby:
41+
desc: Run the ruby test suite.
42+
run: make test-ruby

‎dml_events.go‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -423,6 +423,10 @@ func appendEscapedValue(buffer []byte, value interface{}, column schema.TableCol
423423

424424
switch v := value.(type) {
425425
case string:
426+
// since https://github.com/go-mysql-org/go-mysql/pull/658/files merged, go-mysql returns JSON events as a string, but we would prefer them as []byte for consistency with other types
427+
if column.Type == schema.TYPE_JSON {
428+
return appendEscapedBuffer(buffer, []byte(v), true)
429+
}
426430
var rightPadLengthForBinaryColumn int = 0
427431
// see appendEscapedString() for details why we need special
428432
// handling of BINARY column types
@@ -432,7 +436,8 @@ func appendEscapedValue(buffer []byte, value interface{}, column schema.TableCol
432436

433437
return appendEscapedString(buffer, v, rightPadLengthForBinaryColumn)
434438
case []byte:
435-
return appendEscapedBuffer(buffer, v, column.Type == schema.TYPE_JSON)
439+
// schema type cannot be JSON at this point because all JSON results are strings
440+
return appendEscapedBuffer(buffer, v, false)
436441
case bool:
437442
if v {
438443
return append(buffer, '1')

‎docker-compose.yml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ version: "3"
22
services:
33
mysql-1:
44
image: percona:5.7
5-
platform: linux/x86_64
65
command: --server-id=1
76
--log-bin=mysql-bin
87
--max-binlog-size=4096
@@ -18,14 +17,14 @@ services:
1817
--binlog-rows-query-log-events=ON
1918
environment:
2019
MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
20+
MYSQL_ROOT_HOST: "%"
2121
volumes:
2222
- /var/lib/mysql
2323
ports:
2424
- "29291:3306"
2525

2626
mysql-2:
2727
image: percona:5.7
28-
platform: linux/x86_64
2928
command: --server-id=2
3029
--log-bin=mysql-bin
3130
--binlog-format=ROW
@@ -40,6 +39,7 @@ services:
4039
--binlog-rows-query-log-events=ON
4140
environment:
4241
MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
42+
MYSQL_ROOT_HOST: "%"
4343
volumes:
4444
- /var/lib/mysql
4545
ports:
@@ -61,6 +61,7 @@ services:
6161
--binlog-rows-query-log-events=ON
6262
environment:
6363
MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
64+
MYSQL_ROOT_HOST: "%"
6465
volumes:
6566
- /var/lib/mysql
6667
ports:

‎docker-compose_8.0.yml‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
version: "3"
2+
services:
3+
mysql-1:
4+
image: docker.io/mysql/mysql-server:8.0
5+
command: --server-id=1
6+
--log-bin=mysql-bin
7+
--max-binlog-size=4096
8+
--binlog-format=ROW
9+
--sync-binlog=1
10+
--log-slave-updates=ON
11+
--gtid-mode=ON
12+
--enforce-gtid-consistency=ON
13+
--character-set-server=utf8mb4
14+
--collation-server=utf8mb4_unicode_ci
15+
--max-connections=1000
16+
--read-only=OFF
17+
--binlog-rows-query-log-events=ON
18+
environment:
19+
MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
20+
MYSQL_ROOT_HOST: "%"
21+
volumes:
22+
- /var/lib/mysql
23+
ports:
24+
- "29291:3306"
25+
26+
mysql-2:
27+
image: docker.io/mysql/mysql-server:8.0
28+
command: --server-id=2
29+
--log-bin=mysql-bin
30+
--binlog-format=ROW
31+
--max-binlog-size=4096
32+
--sync-binlog=1
33+
--log-slave-updates=ON
34+
--gtid-mode=ON
35+
--enforce-gtid-consistency=ON
36+
--character-set-server=utf8mb4
37+
--collation-server=utf8mb4_unicode_ci
38+
--max-connections=1000
39+
--binlog-rows-query-log-events=ON
40+
environment:
41+
MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
42+
MYSQL_ROOT_HOST: "%"
43+
volumes:
44+
- /var/lib/mysql
45+
ports:
46+
- "29292:3306"
47+
48+
mysql-3:
49+
image: docker.io/mysql/mysql-server:8.0
50+
command: --server-id=3
51+
--log-bin=mysql-bin
52+
--binlog-format=ROW
53+
--max-binlog-size=4096
54+
--sync-binlog=1
55+
--log-slave-updates=ON
56+
--gtid-mode=ON
57+
--enforce-gtid-consistency=ON
58+
--character-set-server=utf8mb4
59+
--collation-server=utf8mb4_unicode_ci
60+
--max-connections=1000
61+
--binlog-rows-query-log-events=ON
62+
environment:
63+
MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
64+
MYSQL_ROOT_HOST: "%"
65+
volumes:
66+
- /var/lib/mysql
67+
ports:
68+
- "29293:3306"

‎examples/gh-285/bugreport.sh‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,6 @@
22

33
set -e
44

5-
docker-compose up -d
6-
75
mysql -h 127.0.0.1 -u root -P 29291 -e 'DROP DATABASE IF EXISTS `abc`'
86
mysql -h 127.0.0.1 -u root -P 29292 -e 'DROP DATABASE IF EXISTS `abc`'
97

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ require (
66
github.com/DataDog/datadog-go v4.8.2+incompatible
77
github.com/Masterminds/squirrel v0.0.0-20180620232226-b127ed9be034
88
github.com/Microsoft/go-winio v0.5.0 // indirect
9-
github.com/go-mysql-org/go-mysql v1.3.0
9+
github.com/go-mysql-org/go-mysql v1.4.1-0.20220112102103-b3f1a27311d8
1010
github.com/go-sql-driver/mysql v1.5.0
1111
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db
1212
github.com/gorilla/context v1.1.1 // indirect

‎go.sum‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
2323
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
2424
github.com/go-mysql-org/go-mysql v1.3.0 h1:lpNqkwdPzIrYSZGdqt8HIgAXZaK6VxBNfr8f7Z4FgGg=
2525
github.com/go-mysql-org/go-mysql v1.3.0/go.mod h1:3lFZKf7l95Qo70+3XB2WpiSf9wu2s3na3geLMaIIrqQ=
26+
github.com/go-mysql-org/go-mysql v1.4.1-0.20220112102103-b3f1a27311d8 h1:ViPrQui89RKKSFi0FwonSGGCMksRaYFZm+mu8nQpM84=
27+
github.com/go-mysql-org/go-mysql v1.4.1-0.20220112102103-b3f1a27311d8/go.mod h1:3lFZKf7l95Qo70+3XB2WpiSf9wu2s3na3geLMaIIrqQ=
2628
github.com/go-sql-driver/mysql v1.3.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
2729
github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
2830
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=

0 commit comments

Comments
 (0)