Skip to content

Commit f3a0e25

Browse files
Cyb3rWard0gCyb3rWard0g
committed
05312018
Logstash - Added Local Pipeline to the build to allow custom local configurations - updated sysmon config to fix Cyb3rWard0g#63 - removed port exposed in local logstash Dockerfile. It will be pushed to official docker image in the next update - removed logstash init file (not being used anymore) Zeppelin - not available yet - initial draft dockerfile - created spark-defaults file for future zeppelin dockerfile Install Script - incrased minimum memory size required
1 parent bb321d9 commit f3a0e25

File tree

8 files changed

+180
-259
lines changed

8 files changed

+180
-259
lines changed

‎docker-compose.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@ services:
2020
helk-logstash:
2121
image: cyb3rward0g/helk-logstash:6.2.4
2222
container_name: helk-logstash
23+
volumes:
24+
- ./helk-logstash/pipeline:/usr/share/logstash/pipeline
2325
environment:
2426
- "LS_JAVA_OPTS=-Xms2g -Xmx2g"
2527
restart: always

‎helk-logstash/Dockerfile‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,4 @@ LABEL description="Dockerfile base for the HELK Logstash."
1515
# *********** Adding HELK scripts, config files and pipeline configs to ontainer ***************
1616
ADD logstash.yml /usr/share/logstash/config/logstash.yml
1717
ADD pipeline /usr/share/logstash/pipeline
18-
ADD output_templates /usr/share/logstash/output_templates
19-
20-
EXPOSE 5044
18+
ADD output_templates /usr/share/logstash/output_templates

‎helk-logstash/logstash-init‎

Lines changed: 0 additions & 202 deletions
This file was deleted.

‎helk-logstash/pipeline/11-winevent-sysmon-filter.conf‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -298,7 +298,8 @@ filter {
298298
remove_field => "[event_data][UtcTime]"
299299
remove_field => "[event_data][CreationUtcTime]"
300300
remove_field => "[event_data][PreviousCreationUtcTime]"
301-
rename => { "[event_data][User]" => "user"}
301+
remove_field => "[user]"
302+
rename => { "[event_data][User]" => "user_account"}
302303
}
303304
}
304305
}

‎helk-zeppelin/Dockerfile‎

Lines changed: 45 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -10,86 +10,80 @@ LABEL description="Dockerfile base for the HELK Zeppelin."
1010

1111
ENV DEBIAN_FRONTEND noninteractive
1212

13+
USER root
1314
# *********** Installing Prerequisites ***************
14-
# `Z_VERSION` will be updated by `dev/change_zeppelin_version.sh`
15-
ENV Z_VERSION="0.7.3"
15+
ENV Z_VERSION="0.8."
1616
ENV LOG_TAG="[ZEPPELIN_${Z_VERSION}]:" \
17-
Z_HOME="/zeppelin" \
18-
LANG=en_US.UTF-8 \
19-
LC_ALL=en_US.UTF-8
17+
Z_HOME="/zeppelin"
2018

21-
RUN echo "$LOG_TAG update and install basic packages" && \
22-
apt-get -y update && \
23-
apt-get install -y locales && \
24-
locale-gen $LANG && \
25-
apt-get install -y software-properties-common && \
26-
apt -y autoclean && \
27-
apt -y dist-upgrade && \
28-
apt-get install -y build-essential
19+
ENV Z_GID=710
20+
ENV Z_UID=710
21+
ENV Z_USER=zelk
2922

3023
ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
3124
RUN echo "$LOG_TAG Install java8" && \
3225
apt-get -y update && \
33-
apt-get install -y openjdk-8-jdk && \
26+
apt-get install -y git openjdk-8-jdk \
27+
libfontconfig git build-essential chrpath \
28+
libssl-dev libxft-dev libfreetype6 libfreetype6-dev \
29+
libfontconfig1 libfontconfig1-dev python3-pip && \
3430
rm -rf /var/lib/apt/lists/*
3531

36-
# should install conda first before numpy, matploylib since pip and python will be installed by conda
37-
RUN echo "$LOG_TAG Install miniconda2 related packages" && \
38-
apt-get -y update && \
39-
apt-get install -y bzip2 ca-certificates \
40-
libglib2.0-0 libxext6 libsm6 libxrender1 \
41-
git mercurial subversion && \
42-
echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh && \
43-
wget --quiet https://repo.continuum.io/miniconda/Miniconda2-4.3.11-Linux-x86_64.sh -O ~/miniconda.sh && \
44-
/bin/bash ~/miniconda.sh -b -p /opt/conda && \
45-
rm ~/miniconda.sh
46-
ENV PATH /opt/conda/bin:$PATH
32+
# *********** Upgrading PIP ***************
33+
RUN pip3 install --upgrade pip
34+
35+
# *********** Create User *****************
36+
RUN groupadd -r zelk -g ${Z_GID} \
37+
&& useradd -m -s /bin/bash -N -u $Z_UID $Z_USER \
38+
&& chmod g+w /etc/passwd /etc/group \
39+
&& chown -R zelk:zelk /usr/local/ /tmp /usr/bin ${SPARK_HOME}
4740

4841
RUN echo "$LOG_TAG Install python related packages" && \
4942
apt-get -y update && \
50-
apt-get install -y python-dev python-pip && \
5143
apt-get install -y gfortran && \
52-
# numerical/algebra packages
44+
# numerical/algebra packages
5345
apt-get install -y libblas-dev libatlas-dev liblapack-dev && \
5446
# font, image for matplotlib
5547
apt-get install -y libpng-dev libfreetype6-dev libxft-dev && \
5648
# for tkinter
5749
apt-get install -y python-tk libxml2-dev libxslt-dev zlib1g-dev && \
58-
pip install numpy && \
59-
pip install matplotlib
50+
pip3 install numpy && \
51+
pip3 install matplotlib
6052

61-
RUN echo "$LOG_TAG Install R related packages" && \
62-
echo "deb http://cran.rstudio.com/bin/linux/ubuntu xenial/" | tee -a /etc/apt/sources.list && \
63-
gpg --keyserver keyserver.ubuntu.com --recv-key E084DAB9 && \
64-
gpg -a --export E084DAB9 | apt-key add - && \
65-
apt-get -y update && \
66-
apt-get -y install r-base r-base-dev && \
67-
R -e "install.packages('knitr', repos='http://cran.us.r-project.org')" && \
68-
R -e "install.packages('ggplot2', repos='http://cran.us.r-project.org')" && \
69-
R -e "install.packages('googleVis', repos='http://cran.us.r-project.org')" && \
70-
R -e "install.packages('data.table', repos='http://cran.us.r-project.org')" && \
71-
# for devtools, Rcpp
72-
apt-get -y install libcurl4-gnutls-dev libssl-dev && \
73-
R -e "install.packages('devtools', repos='http://cran.us.r-project.org')" && \
74-
R -e "install.packages('Rcpp', repos='http://cran.us.r-project.org')" && \
75-
Rscript -e "library('devtools'); library('Rcpp'); install_github('ramnathv/rCharts')"
53+
# ************** Install PhantpmJS ****************
54+
USER $Z_UID
7655

7756
# ************** Install Maven *********************
7857
ENV MAVEN_VERSION 3.5.3
79-
RUN wget wget -qO- http://www.us.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz | sudo tar xvz -C /usr/local && \
58+
RUN wget -qO- http://www.us.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz | tar xvz -C /usr/local && \
8059
ln -s /usr/local/apache-maven-${MAVEN_VERSION}/bin/mvn /usr/bin/mvn
8160

82-
RUN echo "$LOG_TAG Download Zeppelin binary" && \
83-
wget -O /tmp/zeppelin-${Z_VERSION}-bin-all.tgz http://archive.apache.org/dist/zeppelin/zeppelin-${Z_VERSION}/zeppelin-${Z_VERSION}-bin-all.tgz && \
84-
tar -zxvf /tmp/zeppelin-${Z_VERSION}-bin-all.tgz && \
85-
rm -rf /tmp/zeppelin-${Z_VERSION}-bin-all.tgz && \
86-
mv /zeppelin-${Z_VERSION}-bin-all ${Z_HOME}
61+
USER root
62+
# ************** Install Zeppelin *********************
63+
RUN echo "$LOG_TAG Download Zeppelin" && \
64+
cd /tmp && git clone --branch v0.8.0-rc2 https://github.com/apache/zeppelin.git && \
65+
mv /tmp/zeppelin ${Z_HOME}
66+
67+
RUN chown -R zelk:zelk ${Z_HOME}
68+
69+
USER $Z_UID
70+
71+
RUN cd $Z_HOME && \
72+
mvn clean package -DskipTests -X
73+
74+
# *********** Install CAPS ***************
75+
RUN cd ${Z_HOME} && \
76+
wget https://github.com/opencypher/cypher-for-apache-spark/releases/download/1.0.0-beta7/spark-cypher-1.0.0-beta7-cluster.jar
77+
78+
ADD spark-defaults.conf ${SPARK_HOME}/conf/
8779

80+
USER root
81+
8882
RUN echo "$LOG_TAG Cleanup" && \
8983
apt-get autoclean && \
9084
apt-get clean
9185

92-
EXPOSE 8080
86+
EXPOSE $ZEPPELIN_PORT
9387

9488
WORKDIR ${Z_HOME}
9589
CMD ["bin/zeppelin.sh"]

0 commit comments

Comments
 (0)