sysadmin:storage_setup

You are here: start » sysadmin » storage_setup

SOIT Storage Setup

SOIT Storage Setup

Install Software

Enable EPEL Repo

Get the hosting team to enable the EPEL repo from satellite

RHEL Software

# Authentication support
yum install krb5-workstation openssl-perl

# File sharing
yum install samba.x86_64 samba-client.x86_64 samba-common.x86_64 cifs-utils.x86_64 nfs-utils.x86_64

# Automation support
yum groupinstall "PHP Support" "MySQL Database client"

# Other software (From EPEL)
yum install python-pip php-nrk-Predis php-process php-ldap php-mysql openldap-clients

Change /etc/php.ini

date.timezone = Australia/Sydney

Authentication

SSSD Config

Create /etc/sssd/sssd.conf

[sssd]
#debug_level = 3
config_file_version = 2
services = nss, pam
domains = SHARED.SYDNEY.EDU.AU

[nss]
filter_groups = root
filter_users = root
reconnection_retries = 3
debug_level = 3

[pam]
reconnection_retries = 3

# SHARED Domain
[domain/SHARED.SYDNEY.EDU.AU]
#debug_level = 5
description = SHARED AD Domain
enumerate = false
min_id = 10000
id_provider = ldap
auth_provider = krb5
chpass_provider = krb5
krb5_realm = SHARED.SYDNEY.EDU.AU
dns_discovery_domain = shared.sydney.edu.au

ldap_tls_cacertdir = /etc/openldap/cacerts
ldap_tls_reqcert = demand
ldap_id_use_start_tls = true
ldap_schema = rfc2307bis
ldap_user_search_base = dc=shared,dc=sydney,dc=edu,dc=au
ldap_group_search_base = dc=shared,dc=sydney,dc=edu,dc=au
ldap_default_bind_dn = cn=linuxbind,ou=other users,dc=shared,dc=sydney,dc=edu,dc=au
ldap_default_authtok_type = password
ldap_default_authtok = xxxxxxxxxx

ldap_user_object_class = user
ldap_user_name = sAMAccountName
ldap_user_home_directory = unixHomeDirectory
ldap_user_principal = userPrincipalName

ldap_group_object_class = group
ldap_group_nesting_level = 5

ldap_account_expire_policy = ad
ldap_referrals = false

# THIS IS REQUIRED, BECAUSE BY DEFAULT IN WINDOWS, THE REALM NAME IS LOWERCASE
ldap_force_upper_case_realm = true

[domain/default]
cache_credentials = False

Set the permissions

chmod 0600 /etc/sssd/sssd.conf

Kerberos

Edit /etc/krb5.conf

[logging]
 default = FILE:/var/log/krb5libs.log
 kdc = FILE:/var/log/krb5kdc.log
 admin_server = FILE:/var/log/kadmind.log

[libdefaults]
 default_realm = SHARED.SYDNEY.EDU.AU
 dns_lookup_realm = false
 dns_lookup_kdc = false
 ticket_lifetime = 24h
 renew_lifetime = 7d
 forwardable = true


[realms]
 SHARED.SYDNEY.EDU.AU = {
  kdc = shared-dc-prd-2.shared.sydney.edu.au
  admin_server = shared-dc-prd-2.shared.sydney.edu.au
 }

[domain_realm]
 .shared.sydney.edu.au = SHARED.SYDNEY.EDU.AU
 shared.sydney.edu.au = SHARED.SYDNEY.EDU.AU

NSSwitch

Edit /etc/nsswitch.conf

passwd:     files sss
shadow:     files sss
group:      files sss
netgroup:   nisplus sss

PAM Config

Edit /etc/pam.d/system-auth

auth        required      pam_env.so
# Work around for Bug 1024825 - SSSD - System error in pam_authenticate 
#auth        sufficient    pam_fprintd.so
auth        sufficient    pam_unix.so nullok try_first_pass
auth        requisite     pam_succeed_if.so uid >= 500 quiet
auth        sufficient    pam_sss.so use_first_pass
auth        required      pam_deny.so

account     required      pam_unix.so
account     sufficient    pam_localuser.so
account     sufficient    pam_succeed_if.so uid < 500 quiet
account     required      pam_listfile.so onerr=fail item=group sense=allow file=/etc/login.group.allow
account     [default=bad success=ok user_unknown=ignore] pam_sss.so
account     required      pam_permit.so

password    requisite     pam_cracklib.so try_first_pass retry=3 type=
password    sufficient    pam_unix.so md5 shadow nullok try_first_pass use_authtok
password    sufficient    pam_sss.so use_authtok
password    required      pam_deny.so

session     optional      pam_keyinit.so revoke
session     required      pam_limits.so
session     optional      pam_mkhomedir.so umask=0077
session     [success=1 default=ignore] pam_succeed_if.so service in crond quiet use_uid
session     required      pam_unix.so
session     optional      pam_sss.so

Edit /etc/pam.d/password-auth

auth        required      pam_env.so
auth        sufficient    pam_unix.so nullok try_first_pass
auth        requisite     pam_succeed_if.so uid >= 500 quiet
auth        sufficient    pam_sss.so use_first_pass
auth        required      pam_deny.so

account     required      pam_unix.so
account     sufficient    pam_localuser.so
account     sufficient    pam_succeed_if.so uid < 500 quiet
account     required      pam_listfile.so onerr=fail item=group sense=allow file=/etc/login.group.allow
account     [default=bad success=ok user_unknown=ignore] pam_sss.so
account     required      pam_permit.so

password    requisite     pam_cracklib.so try_first_pass retry=3 type=
password    sufficient    pam_unix.so sha512 shadow nullok try_first_pass use_authtok
password    sufficient    pam_sss.so use_authtok
password    required      pam_deny.so

session     optional      pam_keyinit.so revoke
session     required      pam_limits.so
session     optional      pam_mkhomedir.so umask=0077
session     [success=1 default=ignore] pam_succeed_if.so service in crond quiet use_uid
session     required      pam_unix.so
session     optional      pam_sss.so

SHARED Certificates

Create cert directory

mkdir /etc/openldap/cacerts/
chmod 700 /etc/openldap/cacerts/

Copy in /etc/openldap/cacerts/newsharedca.cer

Copy in /etc/openldap/cacerts/sharedca.cer

Make the cert hash

@TODO: Use c_rehash to do this

cd /etc/openldap/cacerts/
ln -s newsharedca.cer f33ca89c.0
ln -s sharedca.cer 90a16290.0

Change cert perms

Any reason why users can't have read access to these?

chmod -R o+rx /etc/openldap/cacerts/
chmod -R o+r /etc/openldap/cacerts/*

Change /etc/openldap/ldap.conf

TLS_CACERTDIR /etc/openldap/cacerts

Enable + Start sssd

# Enable the service
chkconfig sssd on
 
# Start the service
service sssd start
 
# Start the cache (Use any unikey)
id linuxbind

Security

Logins

Create /etc/login.group.allow

Hosting_Services_Admin
Transition_Services_Admin
{ServerName}
{ServerName}_admin

Root SSH public key

Make SSH directory

mkdir -p /root/.ssh/
chmod 700 /root/.ssh

Create /root/.ssh/authorized_keys

ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAgEA3IrmilVwKGdByOCf2oQlnZnkZUAPRsgNXKXG1RQGGxrox/oXKSwvEJR6T9Ek7tpgbvKwF4x62bt0CcOQFFouyhdw7y+SSa8lbATAc8IqwJoU7o0mr/WLXKXk6v+sbvRYKwcoY+lZUT1G8eiYYvd/7J+hJg7A2ymfdh8OHv58+oNWgJ0T/rn2tCbjMGfu8A0QXpI118gh+VeDPdzPkBUbEhMm5/7nUm+tu37z51M8DZf3w7PY6Y/KlpoyCohdyGko9lYRdJb0W1Z/UThLdZlsXJDM45FvP73dPmGtGr+tHzKe1SA/h24CXy30YyyeyyqW8DkHYHc2ofHbB6TLBbUF90WzHfvVx40h93l1UaY2ViRB6XQidl9Ppngus4W1PD2ALjjFGkvd9X31IZeKKV7gnDSenIY65yBxseNckoApyxVZmSRlOjBo+R/kDufxEQiDwXJGahrfwXP6XbpGVTaLFZP153lJKoMZytMZ6a2epCOv5Q4OX43jcpkqPKjDFxoN6g1JPxVcsMUtBNS3919q+A2pI+x9iInij3IdygLr3PaTyiMNAmv2SXD86wB/0yC9BcdOCDErFzbsJMXmBXyWcEAJRjaXsQ6u2NIskq6kde+DV2NH3CcDrxnPW5u3+mE49FnsS1Tvg7o0kxq97hNbUqtXCx3xpzz4HSZncS0GG0E= USyd Storage

Set keys permissions

chmod 600 /root/.ssh/authorized_keys

Sudoers

Create /etc/sudoers.d/00admins

%Hosting_Services_Admin ALL=(ALL)       ALL
%soit-fs-pro-4_admin ALL=(ALL)       ALL
%Transition_Services_Admin ALL=(ALL)       ALL

Apply goodly perms

chmod 0440 /etc/sudoers.d/00admins

add the include to /etc/sudoers (Needs '#' prefix)

#includedir /etc/sudoers.d

NFS Exports

Edit /etc/sysconfig/nfs

LOCKD_TCPPORT=32803
LOCKD_UDPPORT=32769
MOUNTD_PORT=892

Edit /etc/exports

# Add a line for each UStorClient

/srv/ustor/           10.65.136.80/32(rw,sync,no_subtree_check)

Enable + Start NFS services

chkconfig nfs on
chkconfig rpcbind on
service nfs restart
service rpcbind restart

System Config

Add to /etc/system/limits.conf

# Increase max_open_files limit to match Windows
*               -       nofile          16384

Firewall

Edit /etc/sysconfig/iptables

NFS sources need to point to the UStorMaster server

*filter
:INPUT ACCEPT [0:0]
:FORWARD ACCEPT [0:0]
:OUTPUT ACCEPT [156:20240]
:HI-SYDNET - [0:0]
:HI-INFRA - [0:0]

# Normal stuff
-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
-A INPUT -p icmp -j ACCEPT
-A INPUT -i lo -j ACCEPT

# ssh, http, https, mysql
-A INPUT -p tcp -m state --state NEW -m tcp --dport 22 -j ACCEPT
-A INPUT -p tcp -m state --state NEW -m tcp --dport 80 -j ACCEPT
-A INPUT -p tcp -m state --state NEW -m tcp --dport 443 -j ACCEPT
-A INPUT -p tcp -m state --state NEW -m tcp --dport 3306 -j ACCEPT

# nfs
-A INPUT -s 10.65.136.80/32 -p tcp -m state --state NEW -m multiport --dports 111,892,2049,32803 -j ACCEPT
-A INPUT -s 10.65.136.80/32 -p udp -m state --state NEW -m multiport --dports 111,892,2049,32769 -j ACCEPT

# samba
-A INPUT -p tcp -m state --state NEW -m multiport --dports 137,138,139,445 -j HI-SYDNET
-A INPUT -p udp -m state --state NEW -m multiport --dports 137,138,139,445 -j HI-SYDNET

# Usyd Stuff
-A INPUT -j HI-INFRA

# reject everything else
-A INPUT -j REJECT --reject-with icmp-host-prohibited

# USyd network
-A HI-SYDNET -s 10.0.0.0/255.0.0.0 -j ACCEPT 
-A HI-SYDNET -s 172.16.0.0/255.240.0.0 -j ACCEPT 
-A HI-SYDNET -s 129.78.0.0/255.255.0.0 -j ACCEPT

# USyd Infrastructure
-A HI-INFRA -s 172.18.39.1  -p tcp -m tcp --dport 28002 -j ACCEPT
-A HI-INFRA -s 172.17.12.188 -p tcp -m state --state NEW -m multiport --dports 7005,7006,17005,17006,13705 -j ACCEPT
-A HI-INFRA -s 172.17.12.187 -p tcp -m state --state NEW -m multiport --dports 7005,7006,17005,17006,13705 -j ACCEPT
-A HI-INFRA -s 172.17.12.202 -p tcp -m state --state NEW -m multiport --dports 7005,7006,17005,17006,13705 -j ACCEPT
-A HI-INFRA -s 172.17.12.138 -p tcp -m state --state NEW -m multiport --dports 7005,7006,17005,17006,13705 -j ACCEPT
-A HI-INFRA -s 172.17.12.139 -p tcp -m state --state NEW -m multiport --dports 7005,7006,17005,17006,13705 -j ACCEPT
-A HI-INFRA -s 172.16.71.11 -p tcp -m state --state NEW -m multiport --dports 7005,7006,17005,17006,13705 -j ACCEPT
-A HI-INFRA -s 172.16.245.3 -p tcp -m tcp --dport 3181 -j ACCEPT
-A HI-INFRA -s 172.17.196.59 -p tcp -m tcp --dport 3181 -j ACCEPT
-A HI-INFRA -s 172.17.185.113 -p tcp -m tcp --dport 3181 -j ACCEPT
-A HI-INFRA -s 172.17.185.114 -p tcp -m tcp --dport 3181 -j ACCEPT
-A HI-INFRA -s 172.17.185.117 -p tcp -m tcp --dport 3181 -j ACCEPT
-A HI-INFRA -s 172.17.196.116 -p tcp -m tcp --dport 3181 -j ACCEPT
-A HI-INFRA -s 172.17.12.201 -p tcp -m tcp --dport 3181 -j ACCEPT
-A HI-INFRA -s 172.17.185.102 -p tcp -m tcp --dport 3181 -j ACCEPT
-A HI-INFRA -s 172.20.0.13 -p tcp -m tcp --dport 9898 -j ACCEPT


#drop windows 2012 servers (Not currently compat with samba)
-A OUTPUT -p udp -d 172.16.210.2 -j DROP
-A OUTPUT -p tcp -d 172.16.210.2 -j DROP
-A OUTPUT -p udp -d 172.16.210.15 -j DROP
-A OUTPUT -p tcp -d 172.16.210.15 -j DROP
-A OUTPUT -p udp -d 172.16.210.99 -j DROP
-A OUTPUT -p tcp -d 172.16.210.99 -j DROP
-A OUTPUT -p udp -d 10.83.32.5 -j DROP
-A OUTPUT -p tcp -d 10.83.32.5 -j DROP
-A OUTPUT -p udp -d 10.83.32.6 -j DROP
-A OUTPUT -p tcp -d 10.83.32.6 -j DROP
COMMIT

File Services

Setup Samba

Create /etc/samba/smb.conf

[global]                                                                                                                            
workgroup = SHARED
server string = soit-fs-pro-4

# Logging
log file = /var/log/samba/samba.log
log level = 2
max log size = 1000
syslog = 0

# Security
security = ADS                                                                                                                      
realm = SHARED.SYDNEY.EDU.AU                                                                                                        
encrypt passwords = yes                                                                                                             
template shell = /bin/bash
server signing = auto
client signing = auto
map untrusted to domain = yes

time server = no
dns proxy = no

# General share settings
usershare allow guests = yes                                           
guest account = nobody                                                  

follow symlinks = yes
unix extensions = yes

# Make samba faster
strict locking = no
socket options = IPTOS_LOWDELAY TCP_NODELAY
min receivefile size = 16384
use sendfile = true
aio read size = 16384
aio write size = 16384
aio write behind = false

# Printing stuff
load printers = no
printing = bsd
printcap name = /dev/null
disable spoolss = yes

# Share definitions
include = /etc/samba/includes.conf

Create Default Directories

mkdir -p /srv/ustor
mkdir -p /srv/ustordata/home
chgrp linuxusers /srv/ustordata/home
mkdir -p /srv/ustordata/group
chgrp linuxusers /srv/ustordata/group

Disk partitions

# Partition the disk
parted /dev/sdb
#(parted) mklabel gpt
#(parted) mkpart primary 0% 100%
#(parted) quit

# Create the 2 lvms required
pvcreate /dev/sdb1
vgcreate -s 32M vgdata /dev/sdb1
lvcreate -l 50%FREE -n lvhome vgdata
lvcreate -l 100%FREE -n lvgroup vgdata

# Format the disks with ext4
mkfs.ext4 /dev/mapper/vgdata-lvgroup
mkfs.ext4 /dev/mapper/vgdata-lvhome

Add to end of /etc/fstab

ustor00.it.usyd.edu.au:/srv/ustor  /srv/ustor nfs  defaults,nfsvers=3,tcp 0 0

/dev/mapper/vgdata-lvgroup  /srv/ustordata/group/  ext4  defaults,acl,grpquota,noatime,nodiratime,data=writeback,barrier=0,nobh 1 2
/dev/mapper/vgdata-lvhome  /srv/ustordata/home/  ext4  defaults,acl,usrquota,noatime,nodiratime,data=writeback,barrier=0,nobh 1 2

Mount the lvms

mount -a

Create shared directory

mkdir -p /srv/ustordata/group/shared
chgrp linuxusers /srv/ustordata/group/shared

Share Definitions

Make smb.d

mkdir -p /etc/samba/smb.d/

Create 'homes' share /etc/samba/smb.d/homes.conf

[homes]
        comment = UStor Home Drives
        path = /srv/ustordata/home/%S
        browseable = no
        read only = no
        writable = yes
        valid users = @linuxusers
        create mask = 0600
        force create mode = 0600
        directory mask = 02700
        force directory mode = 02700
        hide unreadable = yes
        hide unwirtable files = no
        root preexec = /srv/ustor/scripts/smbaction.php connect "%T" "%u" "%S" "%P" "%m" "%I" "%H" "%g" "%a" "%h" "%L" "%$usn"

Create 'homes' share /etc/samba/smb.d/shared.conf

[shared]
        comment = UStor Shared Data
        path = /srv/ustordata/group/shared
        browseable = yes
        read only = no
        writable = yes
        valid users = @linuxusers
        create mask = 0660
        force create mode = 0660
        directory mask = 02770
        force directory mode = 02770
        root preexec = /srv/ustor/scripts/smbaction.php connect "%T" "%u" "%S" "%P" "%m" "%I" "%H" "%g" "%a" "%h" "%L" "%$usn"

Link the smbincludes creation file into cron.hourly

#smbconf.cron
#!/bin/bash
 
ls /etc/samba/smb.d/* | sed -e 's/^/include = /' > /etc/samba/includes.conf

ln -s /srv/ustor/scripts/smbconf.cron /etc/cron.hourly/smbconf.cron

Join to the SHARED domain

net ads join -U svc_sit_ustor@SHARED.SYDNEY.EDU.AU%xxxxxxxx createcomputer=Groups/SOIT/Servers osName=RHEL osVer=6.6

@TODO: Look into this error

Failed to join domain: failed to set machine spn: Constraint violation

Config SELinux

* Not required since disabling SELinux *

# Enable home dirs
setsebool -P samba_enable_home_dirs on
 
# Allow preexec from the scripts dir
chcon -t samba_unconfined_script_exec_t /srv/ustor/scripts/
 
# And the logs
chcon -R -t samba_unconfined_script_exec_t /srv/ustor/logs/
 
# Allow share for shared directory
chcon -t samba_share_t /srv/ustordata/shared/
 
# Allow home for each home directory
# chcon -t samba_share_t /srv/ustordata/home/pfowler

Enable and Start

chkconfig smb on
chkconfig nmb on
service smb start
service nmb start

Enable Quota'ing

Fix the SELinux Policies

# SELinux now Disabled
# Mimic /home/ policies
#semanage fcontext -a -e /home /srv/ustordata
# Apply it
#restorecon -R -v /srv/ustordata

Setup the Quota

quotacheck -cg  /srv/ustordata/group
quotacheck -cu  /srv/ustordata/home
quotaon -avug

Setting quota's

# 300 Soft, 500 Hard
setquota -u pfowler 307200 512000 0 0 -a /srv/ustordata/home
setquota -g sharedgroupname 307200 512000 0 0 -a /srv/ustordata/group

UStor Config

Create 'UStor Name' variable

Create /etc/profile.d/usn.sh

export usn=soit-fs-pro-4

Make it executable

chmod a+rx /etc/profile.d/usn.sh

Trace: • storage_setup

Table of Contents