Mattermost, Inc.

Gitlab-Mattermost behind AWS ALB does not work, i/o timeout

For troubleshooting questions, please post in the following format:

Summary

Hi all,
Here is my trouble when I try to login Mattermot with Gitlab account.

Error

Token request failed

[Back to GitLab Mattermost](https://mattermost.xxx.vn/)

Show on logs:
==> /var/log/gitlab/mattermost/mattermost.log <==
{“level”:“error”,“ts”:1580810926.2013972,“caller”:“web/oauth.go:259”,“msg”:“AuthorizeOAuthUser: Token request failed, Post http://gitlab.supremetech.vn/oauth/token: dial tcp 54.210.131.230:80: i/o timeout”}

==> /var/log/gitlab/mattermost/current <==
2020-02-04_10:08:46.20147 {"level":"error","ts":1580810926.2013972,"caller":"web/oauth.go:259","msg":"AuthorizeOAuthUser: Token request failed, Post http://gitlab.supremetech.vn/oauth/token: dial tcp 54.210.131.230:80: i/o timeout"}

Steps to reproduce

How can we reproduce the issue (what version are you using?)

  • On AWS, Create instance t2.medium with Amazonlinux 2 image. Create Application Loadbalancer (Listener port80 and port443)

  • User docker-compose to start service, here is docker-compose file content:
    version: “3.7”
    services:
    # db:
    # restart: always
    # image: postgres:12.1
    # container_name: “db”
    # volumes:
    # # Mount EFS volume
    # - /home/ec2-user/efs/gitlab/db/data:/var/lib/postgresql/data:Z
    # - /etc/localtime:/etc/localtime:ro
    # environment:
    # - POSTGRES_USER=gitlab
    # - POSTGRES_PASSWORD=pASsw0rd
    # - POSTGRES_DB=st_gitlab_production
    # ports:
    # - “5432:5432”

    gitlab:
      container_name: gitlab
      image: "gitlab/gitlab-ce:12.5.9-ce.0"
      restart: always
      #hostname: "gitlab.supremetech.vn"
      environment:
        GITLAB_OMNIBUS_CONFIG: |
          # Gitlab basic configuration
          letsencrypt['enable'] = false
          letsencrypt['contact_emails'] = ['gitlab@example.com']
          external_url 'https://gitlab.supremetech.vn'
          # Prevent GitLab from starting if NFS data mounts are not available (not working for docker)
          # high_availability['mountpoint'] = '/var/opt/gitlab/git-data'
          # Ensure UIDs and GIDs match between servers for permissions via NFS
          user['uid'] = 9000
          user['gid'] = 9000
          web_server['uid'] = 9001
          web_server['gid'] = 9001
          registry['uid'] = 9002
          registry['gid'] = 9002
          #
          gitlab_rails['registry_enabled'] = false
          gitlab_rails['time_zone'] = 'Asia/Ho_Chi_Minh'
          gitlab_rails['lfs_enabled'] = true
          gitlab_rails['initial_root_password'] = 'root@!2vWzOb15u'
          gitlab_rails['gitlab_shell_ssh_port'] = 2224
          # Change URL for clone via SSH
          gitlab_rails['gitlab_ssh_host'] = 'gitlab-ssh.supremetech.vn'
          # OmniAuth Configuration
          gitlab_rails['omniauth_enabled'] = true
          gitlab_rails['omniauth_external_providers'] = ['saml']
          gitlab_rails['omniauth_allow_single_sign_on'] = ['saml']
          gitlab_rails['omniauth_sync_email_from_provider'] = 'saml'
          gitlab_rails['omniauth_block_auto_created_users'] = false
          gitlab_rails['omniauth_auto_link_ldap_user'] = false
          gitlab_rails['omniauth_sync_profile_from_provider'] = ['saml']
          gitlab_rails['omniauth_sync_profile_attributes'] = ['email']
          gitlab_rails['omniauth_auto_link_saml_user'] = true
          # Uncomment this once you 100% ready to use SSO
          # gitlab_rails['omniauth_auto_sign_in_with_provider'] = 'saml'
          gitlab_rails['omniauth_providers'] = [
            {
              # G-Suite
              'name': 'saml',
              'args': {
                      'assertion_consumer_service_url': 'https://gitlab.supremetech.vn/users/auth/saml/callback',
                      'idp_cert_fingerprint': '6F:EB:CA:49:7E:2A:0F:7F:AE:F6:81:89:D5:74:45:C2:54:EB:49:8C',
                      'idp_sso_target_url': 'https://accounts.google.com/o/saml2/idp?idpid=C04iisy27',
                      'issuer': 'https://gitlab.supremetech.vn',
                      'name_identifier_format': 'urn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress',
                      'attribute_statements': { 'email': ['emailAddress'] }
                    },
              'label': 'G Suite'
            }
          ]
          # Email Settings
          gitlab_rails['gitlab_email_from'] = 'bach@supremetech.vn'
          gitlab_rails['gitlab_email_display_name'] = 'Gitlab Supremetech'
          gitlab_rails['gitlab_email_reply_to'] = 'bach@supremetech.vn'
          gitlab_rails['gitlab_email_subject_suffix'] = 'Example GitLab'
          # Disable the built-in Postgres
          postgresql['enable'] = false
          # Recommend value is 1/4 of total RAM, up to 14GB.
          postgresql['shared_buffers'] = '2GB'
          # PostgreSQl database configuration
          gitlab_rails['db_adapter'] = 'postgresql'
          gitlab_rails['db_encoding'] = 'utf8'
          gitlab_rails['db_host'] = 'write.db.local'
          gitlab_rails['db_port'] = 5432
          gitlab_rails['db_database'] = 'st_gitlab_production'
          gitlab_rails['db_username'] = 'master'
          gitlab_rails['db_password'] = 'pASsw0rd'
          # Gmail SMTP configuration
          gitlab_rails['smtp_enable'] = true
          gitlab_rails['smtp_address'] = 'smtp.gmail.com'
          gitlab_rails['smtp_port'] = 587
          gitlab_rails['smtp_user_name'] = 'bach@supremetech.vn'
          gitlab_rails['smtp_password'] = 'kuwqxahgbxlvmytl'
          gitlab_rails['smtp_domain'] = 'smtp.gmail.com'
          gitlab_rails['smtp_authentication'] = 'login'
          gitlab_rails['smtp_enable_starttls_auto'] = true
          gitlab_rails['smtp_tls'] = false
          gitlab_rails['smtp_openssl_verify_mode'] = 'peer'
          # Nginx config
          nginx['redirect_http_to_https'] = false
          # Fix git clone https via ALB
          nginx['listen_port'] = 80
          nginx['listen_https'] = false
          nginx['proxy_set_headers'] = {
            "X-Forwarded-Proto" => "https",
            "X-Forwarded-Ssl" => "on"
          }
    
          # MatterMost Configuration
          # mattermost['enable'] = true# gitlab-ce/config/gitlab.rb
          mattermost_external_url 'https://mattermost.supremetech.vn'
          mattermost['service_site_url'] = 'https://mattermost.supremetech.vn'
          # mattermost['service_use_ssl'] = false
          #
          # mattermost['service_use_ssl'] = true
          mattermost_nginx['listen_port'] = 80
          mattermost_nginx['listen_https'] = false
          mattermost_nginx['proxy_set_headers'] = {
            'X-Forwarded-Proto' => 'https',
            'X-Forwarded-Ssl' => 'on'
          }
          # GitLab as the only external authentication source
          mattermost['email_enable_sign_up_with_email'] = false
          mattermost['email_enable_sign_in_with_email'] = false
          mattermost['email_enable_sign_in_with_username'] = false
          # Configure an e-mail address and SMTP server for Mattermost
          mattermost['email_send_email_notifications'] = true
          mattermost['email_require_email_verification'] = false
          mattermost['email_smtp_username'] = "bach@supremetech.vn"
          mattermost['email_smtp_password'] = "kuwqxahgbxlvmytl"
          mattermost['email_smtp_server'] = "smtp.gmail.com"
          mattermost['email_smtp_port'] = "587" # 587
          mattermost['email_connection_security'] = 'TLS' # 'TLS', 'STARTTLS' or nil
          mattermost['email_feedback_name'] = "GitLab Mattermost"
          mattermost['email_feedback_email'] = "bach@supremetech.vn"
          # E-mail batching allowing users to control how often they receive notifications
    
          mattermost['email_enable_batching'] = true
          # Config DB for mattermost
          mattermost['sql_driver_name'] = 'postgres'
          mattermost['sql_data_source'] = "user=master host=write.db.local port=5432 dbname=mattermost password=pASsw0rd"
          # Gitlab Mattermost
          mattermost['gitlab_enable'] = true
          mattermost['gitlab_id'] = "86ef2661bbc88f59b6963e80b648b8c3ebdc8b4628342f28c14cebd5a6d449e3"
          mattermost['gitlab_secret'] = "cc41afff7dbff5c1dabcf207b78318b68bf1d117577c6bd3bb5b6ce2c13643d9"
          mattermost['gitlab_scope'] = ""
          mattermost['gitlab_auth_endpoint'] = "https://gitlab.supremetech.vn/oauth/authorize"
          mattermost['gitlab_token_endpoint'] = "https://gitlab.supremetech.vn/oauth/token"
          mattermost['gitlab_user_api_endpoint'] = "https://gitlab.supremetech.vn/api/v4/user"
          # fix token error
          mattermost['service_enable_insecure_outgoing_connections'] = true
          mattermost['uid'] = 1234
          mattermost['gid'] = 1234
      ports:
        - "80:80"
        - "443:443"
        - "2224:22"
      volumes:
        # Mount EFS volume
        - "/home/ec2-user/efs/gitlab/config:/etc/gitlab"
        - "/var/log/gitlab:/var/log/gitlab"
        - "/home/ec2-user/efs/gitlab/data:/var/opt/gitlab"
    
  • run: docker-compose up -d

Observed behavior

What did you see happen? Please include relevant error messages and/or screenshots.
Homepage of mattermost:
https://prnt.sc/qxbk2r

After click Gitlab Signle Sign On:
https://prnt.sc/qxbl47

Expected behavior

Describe your issue in detail

docker logs -f gitlab result:

==> /var/log/gitlab/mattermost/current <==
2020-02-04_09:47:29.77464 {"level":"info","ts":1580809649.7746255,"caller":"app/server.go:220","msg":"Loaded config","source":"file:///var/opt/gitlab/mattermost/config.json"}
2020-02-04_09:47:29.79643 {"level":"info","ts":1580809649.79637,"caller":"sqlstore/post_store.go:1356","msg":"Post.Message has size restrictions","max_characters":16383,"max_bytes":65535}
2020-02-04_09:47:29.86422 {"level":"info","ts":1580809649.8641675,"caller":"jobs/workers.go:68","msg":"Starting workers"}
2020-02-04_09:47:29.86488 {"level":"info","ts":1580809649.8642733,"caller":"app/server.go:423","msg":"Starting Server..."}
2020-02-04_09:47:29.86489 {"level":"info","ts":1580809649.8643892,"caller":"app/server.go:489","msg":"Server is listening on 127.0.0.1:8065"}
2020-02-04_09:47:29.86579 {"level":"info","ts":1580809649.8657637,"caller":"jobs/schedulers.go:72","msg":"Starting schedulers."}
2020-02-04_09:47:29.87263 {"level":"info","ts":1580809649.8725827,"caller":"app/web_hub.go:75","msg":"Starting 4 websocket hubs"}
2020-02-04_09:47:29.87348 {"level":"error","ts":1580809649.8732216,"caller":"web/static.go:26","msg":"Failed to update assets subpath from config","error":"failed to update root.html with subpath /: open /opt/gitlab/embedded/service/mattermost/client/root.html: permission denied","errorVerbose":"open /opt/gitlab/embedded/service/mattermost/client/root.html: permission denied\nfailed to update root.html with subpath /\ngithub.com/mattermost/mattermost-server/utils.UpdateAssetsSubpath\n\t/go/src/github.com/mattermost/mattermost-server/utils/subpath.go:117\ngithub.com/mattermost/mattermost-server/utils.UpdateAssetsSubpathFromConfig\n\t/go/src/github.com/mattermost/mattermost-server/utils/subpath.go:156\ngithub.com/mattermost/mattermost-server/web.(*Web).InitStatic\n\t/go/src/github.com/mattermost/mattermost-server/web/static.go:25\ngithub.com/mattermost/mattermost-server/web.New\n\t/go/src/github.com/mattermost/mattermost-server/web/web.go:39\ngithub.com/mattermost/mattermost-server/cmd/mattermost/commands.runServer\n\t/go/src/github.com/mattermost/mattermost-server/cmd/mattermost/commands/server.go:83\ngithub.com/mattermost/mattermost-server/cmd/mattermost/commands.serverCmdF\n\t/go/src/github.com/mattermost/mattermost-server/cmd/mattermost/commands/server.go:53\ngithub.com/spf13/cobra.(*Command).execute\n\t/go/src/github.com/mattermost/mattermost-server/vendor/github.com/spf13/cobra/command.go:826\ngithub.com/spf13/cobra.(*Command).ExecuteC\n\t/go/src/github.com/mattermost/mattermost-server/vendor/github.com/spf13/cobra/command.go:914\ngithub.com/spf13/cobra.(*Command).Execute\n\t/go/src/github.com/mattermost/mattermost-server/vendor/github.com/spf13/cobra/command.go:864\ngithub.com/mattermost/mattermost-server/cmd/mattermost/commands.Run\n\t/go/src/github.com/mattermost/mattermost-server/cmd/mattermost/commands/root.go:15\nmain.main\n\t/go/src/github.com/mattermost/mattermost-server/cmd/mattermost/main.go:30\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:200\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1337"}
2020-02-04_09:51:50.54255 {"level":"error","ts":1580809910.542492,"caller":"web/oauth.go:259","msg":"AuthorizeOAuthUser: Token request failed, Post http://gitlab.xxx.xxx/oauth/token: dial tcp 34.226.43.157:80: i/o timeout"}
2020-02-04_09:54:22.49820 {"level":"error","ts":1580810062.4981291,"caller":"web/oauth.go:259","msg":"AuthorizeOAuthUser: Token request failed, Post http://gitlab.xxx.xxx/oauth/token: dial tcp 54.210.131.230:80: i/o timeout"}

==> /var/log/gitlab/mattermost/mattermost.log <==
{"level":"error","ts":1580810824.368601,"caller":"web/oauth.go:259","msg":"AuthorizeOAuthUser: Token request failed, Post http://gitlab.xxx.xxx/oauth/token: dial tcp 54.210.131.230:80: i/o timeout"}

==> /var/log/gitlab/mattermost/current <==
2020-02-04_10:07:04.36868 {"level":"error","ts":1580810824.368601,"caller":"web/oauth.go:259","msg":"AuthorizeOAuthUser: Token request failed, Post http://gitlab.xxx.xxx/oauth/token: dial tcp 54.210.131.230:80: i/o timeout"}

==> /var/log/gitlab/mattermost/mattermost.log <==
{"level":"error","ts":1580810926.2013972,"caller":"web/oauth.go:259","msg":"AuthorizeOAuthUser: Token request failed, Post http://gitlab.xxx.xxx/oauth/token: dial tcp 54.210.131.230:80: i/o timeout"}

==> /var/log/gitlab/mattermost/current <==
2020-02-04_10:08:46.20147 {"level":"error","ts":1580810926.2013972,"caller":"web/oauth.go:259","msg":"AuthorizeOAuthUser: Token request failed, Post http://gitlab.xxx.xxx/oauth/token: dial tcp 54.210.131.230:80: i/o timeout"}

==> /var/log/gitlab/mattermost/mattermost.log <==
{"level":"error","ts":1580813729.111235,"caller":"web/oauth.go:259","msg":"AuthorizeOAuthUser: Token request failed, Post http://gitlab.xxx.xxx/oauth/token: dial tcp 54.210.131.230:80: i/o timeout"}

==> /var/log/gitlab/mattermost/current <==
2020-02-04_10:55:29.11131 {"level":"error","ts":1580813729.111235,"caller":"web/oauth.go:259","msg":"AuthorizeOAuthUser: Token request failed, Post http://gitlab.xxx.xxx/oauth/token: dial tcp 54.210.131.230:80: i/o timeout"}